Handle big HttpEntity response

[mserioli]

Hi.

I make a call to my restful controller that receive a very big amount of json data.

How can I store this json into a file? I want to read in sequence each item using custom jackson object mapper..

Thank you!
Marco

[shredder]

This is what I used but be aware it might crash some time.

(function() {
app = app || {};
app.ui = {};

var ROOT_DIR = Ti.Filesystem.applicationDataDirectory;
var SEP = Titanium.Filesystem.getSeparator();

// Initialize database
var db = Titanium.Database.install('contentTest.sqlite','co ntentTest');
db.execute('DROP TABLE IF EXISTS test');
db.execute('CREATE TABLE IF NOT EXISTS test (test_id INTEGER PRIMARY KEY, test_content BLOB)');
db.close();


var countTest = 100;


app.ui.createDBFileTestWindow = function(){
var win = Titanium.UI.createWindow();

var view = Titanium.UI.createView({backgroundColor:'#fff', layout: 'vertical'});

var content = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaa';


var TestFileCallback = function(count){

var filePath = ROOT_DIR + SEP + 'testFile-' + count + '.txt';
var f = Titanium.Filesystem.getFile(filePath);

f.write(content);
f = null;

f = Titanium.Filesystem.getFile(filePath);
var blob = f.read();
var readText = blob.text;

f = null;
blob = null;

return readText;
}



var testDBCallback = function(count){

var db = Titanium.Database.open('contentTest');
var theData = db.execute('INSERT INTO test (test_id, test_content) VALUES(?,?)',count,content);

var resultset = db.execute('SELECT test_content FROM test WHERE test_id = ' + count);
var readText = resultset.fieldByName("test_content");

resultset.close();

db = null;
theData = null;
resultset = null;


return readText;

}

var buttonFile = Titanium.UI.createButton({title: 'Test File',top: 10,width: 100,height: 50});
buttonFile.addEventListener('click',function(e){
Titanium.API.info("You clicked the File button");
var startTime = new Date().getTime();
var content = null;

var fileCounter = 0;
for (i=0;i<countTest;i++){
var content = TestFileCallback(i);
fileCounter++;
}

var endTime = new Date().getTime();
var diffTime = endTime - startTime;

for (i=0;i<countTest;i++){
var filePath = ROOT_DIR + SEP + 'testFile-' + i + '.txt';
var f = Titanium.Filesystem.getFile(filePath);
f.deleteFile();
f = null;
}

alert('Count: ' + fileCounter + ' Duration: ' + diffTime + ' ms' + ' Content-Length: ' + content.length);

});

var buttonDatabase = Titanium.UI.createButton({title: 'Test Database',top: 10,width: 100,height: 50});
buttonDatabase.addEventListener('click',function(e ){
Titanium.API.info("You clicked the Database button");
var startTime = new Date().getTime();
var content = null;
var dbCounter = 0;
for (i=0;i<countTest;i++){
content = testDBCallback(i);
dbCounter++;
}

var endTime = new Date().getTime();
var diffTime = endTime - startTime;

var db = Titanium.Database.open('contentTest');
for (i=0;i<countTest;i++){
theData = db.execute('DELETE FROM test WHERE test_id = ' + i);
}
db.close();

alert('Count: ' + dbCounter + ' Duration: ' + diffTime + ' ms' + ' Content-Length: ' + content.length);
});

view.add(buttonFile);
view.add(buttonDatabase);

win.add(view);

return win;
}

})();

[mserioli]

I've solved.

I save the byte[] of the response by pass a ByteArrayHttpMessageConverter to the RestTemplate and then streaming the result of restTemplate.exchange(url, HttpMethod.GET, requestEntity, byte[].class) to a file.

Then with jackson I create a parser for the file:
final JsonParser jp = new ObjectMapper().getJsonFactory().createJsonParser(f ile);

In this way I can scan sequentially a very big json file avoiding to map it to a List and to get out of memory exceptions..

In my example this json file contains ~ 56000 orders, with about 10 each field.

Hope this help someone!
Marco