How To Make Synchronous Http Calls Using Promises In Node.js
I would like to iterate thru an array of students and make http call for each of them and parse the response and insert into mongodb, so I would like to do this for each student on
Solution 1:
First off, multiple nested operations are going to be a lot easier to code and handle errors reliably if you use promises for all async operations. That means learning how to use the promises that are built into your database (I assume you're using mongoose) and then wrapping any other async operations to use promises. Here are a couple links on using promises with mongoose:
Switching to use promises in Mongoose
Plugging in your own promise library into Mongoose
So, looking at the Mongoose documentation, you can see that .exec() and .save() already return promises so we can use them directly.
Adding this line of code will tell Mongoose to use Q promises (since that's the promise library you show):
// tell mongoose to use Q promises
mongoose.Promise = require('q').Promise;
Then, you need to promisify some operations that don't use promises such as the parsing step:
functionparse(r) {
var deferred = Q.defer();
parser.parseString(r, function(err, results) {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(results);
}
});
return deferred.promise;
}
saveTimeEntry() can easily be written to return a promise, just using the promise support already in your database:
functionsaveTimeEntry(item) {
returnStudent.findOne({'worksnap.user.user_id': item.user_id[0]}).populate('user').exec().then(function(student) {
student.timeEntries.push(item);
return student.save();
});
}
So, now you have all the right pieces to rewrite the main logic using promises. The key thing to remember here is that if you return a promise from a .then() handler, it chains that promise onto the parent promise. We will use that a lot in your processing. Further, a common design pattern for sequencing promises that are iterating through an array is to use array.reduce() like this:
returnarray.reduce(function(p, item) {
return p.then(function() {
return someAsyncPromiseOperation(item);
});
}, Q());
We will use that structure in a couple of places to rewrite the core logic using promises:
// tell mongoose to use Q promises
mongoose.Promise = require('q').Promise;
iterateThruAllStudents(from, to).then(function() {
// done successfully here
}, function(err) {
// error occurred here
});
functioniterateThruAllStudents(from, to, callback) {
returnStudent.find({status: 'student'}).populate('user').exec().then(function (students) {
// iterate through the students array sequentially
students.reduce(function(p, student) {
return p.then(function() {
if (student.worksnap.user != null) {
var worksnapOptions = {
hostname: 'worksnaps.com',
path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id +
'&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {'Authorization': 'Basic xxx='},
method: 'GET'
};
returnpromisedRequest(worksnapOptions).then(function(response) {
returnparse(response).then(function(results) {
// assuming results.time_entries is an arrayreturn results.time_entries.reduce(function(p, item) {
return p.then(function() {
returnsaveTimeEntry(item);
});
}, Q());
});
});
}
});
}, Q())
});
}
functionparse(r) {
var deferred = Q.defer();
parser.parseString(r, function(err, results) {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(results);
}
});
return deferred.promise;
}
functionsaveTimeEntry(item) {
returnStudent.findOne({'worksnap.user.user_id': item.user_id[0]}).populate('user').exec().then(function(student) {
student.timeEntries.push(item);
return student.save();
});
}
functionpromisedRequest(requestOptions) {
//create a deferred object from Q
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var deferred = Q.defer();
var req = http.request(requestOptions, function (response) {
//set the response encoding to parse json string
response.setEncoding('utf8');
var responseData = '';
//append data to responseData variable on the 'data' event emission
response.on('data', function (data) {
responseData += data;
});
//listen to the 'end' event
response.on('end', function () {
//resolve the deferred object with the responseconsole.log('http call finished');
deferred.resolve(responseData);
});
});
//listen to the 'error' event
req.on('error', function (err) {
//if an error occurs reject the deferred
deferred.reject(err);
});
req.end();
return deferred.promise;
}
One thing this offers that your code does not is that ANY error that occurs will percolate all the way back to the returned promise from iterateThruAllStudents() so no errors are hidden.
And, then cleaning it up some to reduce nested indentation and adding a useful utility function, it looks like this:
// tell mongoose to use Q promises
mongoose.Promise = require('q').Promise;
// create utility function for promise sequencing through an arrayfunctionsequence(array, iterator) {
return array.reduce(function(p, item) {
return p.then(function() {
returniterator(item);
});
}, Q());
}
iterateThruAllStudents(from, to).then(function() {
// done successfully here
}, function(err) {
// error occurred here
});
functioniterateThruAllStudents(from, to, callback) {
returnStudent.find({status: 'student'}).populate('user').exec().then(function (students) {
// iterate through the students array sequentiallyreturnsequence(students, function(item) {
if (student.worksnap.user != null) {
var worksnapOptions = {
hostname: 'worksnaps.com',
path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id +
'&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {'Authorization': 'Basic xxx='},
method: 'GET'
};
returnpromisedRequest(worksnapOptions).then(function(response) {
returnparse(response);
}).then(function(results) {
// assuming results.time_entries is an arrayreturnsequence(results.time_entries, saveTimeEntry);
});
}
});
});
}
functionparse(r) {
var deferred = Q.defer();
parser.parseString(r, function(err, results) {
if (err) {
deferred.reject(err);
} else {
deferred.resolve(results);
}
});
return deferred.promise;
}
functionsaveTimeEntry(item) {
returnStudent.findOne({'worksnap.user.user_id': item.user_id[0]}).populate('user').exec().then(function(student) {
student.timeEntries.push(item);
return student.save();
});
}
functionpromisedRequest(requestOptions) {
//create a deferred object from Q
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var deferred = Q.defer();
var req = http.request(requestOptions, function (response) {
//set the response encoding to parse json string
response.setEncoding('utf8');
var responseData = '';
//append data to responseData variable on the 'data' event emission
response.on('data', function (data) {
responseData += data;
});
//listen to the 'end' event
response.on('end', function () {
//resolve the deferred object with the responseconsole.log('http call finished');
deferred.resolve(responseData);
});
});
//listen to the 'error' event
req.on('error', function (err) {
//if an error occurs reject the deferred
deferred.reject(err);
});
req.end();
//we are returning a promise object//if we returned the deferred object//deferred object reject and resolve could potentially be modified//violating the expected behavior of this functionreturn deferred.promise;
}
Of course, this is a lot of code and I have no way to test it and I've never written Mongoose code before, so there very well could be some goofs in here, but hopefully you can see the general idea and work through any mistakes I might have made.
Solution 2:
Ok I did resolved this problem and if someone in future has this problem here is a solution I came of.
var startDate = newDate("February 20, 2016 00:00:00"); //Start from Februaryvarfrom = newDate(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = newDate(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);
functioniterateThruAllStudents(from, to) {
Student.find({status: 'student'})
.populate('user')
.exec(function (err, students) {
if (err) {
throw err;
}
var counter = 1;
async.eachSeries(students, functioniteratee(student, callback) {
counter++;
if (student.worksnap.user != null) {
console.log('');
console.log('--------------');
console.log(student.worksnap.user.user_id);
var worksnapOptions = {
hostname: 'worksnaps.com',
path: '/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
headers: {
'Authorization': 'Basic xxxxx'
},
method: 'GET'
};
promisedRequest(worksnapOptions)
.then(function (response) { //callback invoked on deferred.resolve
parser.parseString(response, function (err, results) {
var json_string = JSON.stringify(results.time_entries);
var timeEntries = JSON.parse(json_string);
var isEmpty = _.isEmpty(timeEntries); // trueif (isEmpty) {
callback(null);
}
saveTimeEntry(timeEntries).then(function (response) {
console.log('all timeEntries for one student finished....Student: ' + student.worksnap.user.user_id + ' Student Counter: ' + counter);
callback(null);
});
});
}, function (newsError) { //callback invoked on deferred.rejectconsole.log(newsError);
});
} else {
callback(null);
}
});
});
}
functionsaveTimeEntry(timeEntries) {
var deferred = Q.defer();
_.forEach(timeEntries, function (timeEntry) {
_.forEach(timeEntry, function (item) {
Student.findOne({
'worksnap.user.user_id': item.user_id[0]
})
.populate('user')
.exec(function (err, student) {
if (err) {
//throw err;console.log(err);
}
student.timeEntries.push(item);
student.save(function (err) {
if (err) {
console.log(err);
deferred.reject(err);
} else {
//console.log(Math.random());
}
});
});
});
deferred.resolve('finished saving timeEntries for one student...');
});
return deferred.promise;
}
functionpromisedRequest(requestOptions) {
//create a deferred object from Q
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
var deferred = Q.defer();
var req = http.request(requestOptions, function (response) {
//set the response encoding to parse json string
response.setEncoding('utf8');
var responseData = '';
//append data to responseData variable on the 'data' event emission
response.on('data', function (data) {
responseData += data;
});
//listen to the 'end' event
response.on('end', function () {
//resolve the deferred object with the responseconsole.log('http call finished');
deferred.resolve(responseData);
});
});
//listen to the 'error' event
req.on('error', function (err) {
//if an error occurs reject the deferredconsole.log('inside On error.');
console.log(err);
deferred.reject(err);
});
req.end();
//we are returning a promise object//if we returned the deferred object//deferred object reject and resolve could potentially be modified//violating the expected behavior of this functionreturn deferred.promise;
}
Post a Comment for "How To Make Synchronous Http Calls Using Promises In Node.js"