Callback After All Asynchronous Foreach Callbacks Are Completed

Callback after all asynchronous forEach callbacks are completed

Array.forEach does not provide this nicety (oh if it would) but there are several ways to accomplish what you want:

Using a simple counter

function callback () { console.log('all done'); }

var itemsProcessed = 0;

[1, 2, 3].forEach((item, index, array) => {
  asyncFunction(item, () => {
    itemsProcessed++;
    if(itemsProcessed === array.length) {
      callback();
    }
  });
});

(thanks to @vanuan and others) This approach guarantees that all items are processed before invoking the "done" callback. You need to use a counter that gets updated in the callback. Depending on the value of the index parameter does not provide the same guarantee, because the order of return of the asynchronous operations is not guaranteed.

Using ES6 Promises

(a promise library can be used for older browsers):

1. Process all requests guaranteeing synchronous execution (e.g. 1 then 2 then 3)

   function asyncFunction (item, cb) {
     setTimeout(() => {
       console.log('done with', item);
       cb();
     }, 100);
   }
   
   let requests = [1, 2, 3].reduce((promiseChain, item) => {
       return promiseChain.then(() => new Promise((resolve) => {
         asyncFunction(item, resolve);
       }));
   }, Promise.resolve());
   
   requests.then(() => console.log('done'))
   

2. Process all async requests without "synchronous" execution (2 may finish faster than 1)

   let requests = [1,2,3].map((item) => {
       return new Promise((resolve) => {
         asyncFunction(item, resolve);
       });
   })
   
   Promise.all(requests).then(() => console.log('done'));
   

Using an async library

There are other asynchronous libraries, async being the most popular, that provide mechanisms to express what you want.

Edit

---

The body of the question has been edited to remove the previously synchronous example code, so i've updated my answer to clarify.
The original example used synchronous like code to model asynchronous behaviour, so the following applied:

array.forEach is synchronous and so is res.write, so you can simply put your callback after your call to foreach:

  posts.foreach(function(v, i) {
    res.write(v + ". index " + i);
  });

  res.end();

Best way to wait for .forEach() to complete

If there is no asynchronous code inside the forEach, forEach is not asynchronous, for example in this code:

array.forEach(function(item){ 
    //iterate on something 
});
alert("Foreach DONE !");

you will see the alert after forEach finished.

Otherwise (You have something asynchronous inside), you can wrap the forEach loop in a Promise:

var bar = new Promise((resolve, reject) => {
    foo.forEach((value, index, array) => {
        console.log(value);
        if (index === array.length -1) resolve();
    });
});

bar.then(() => {
    console.log('All done!');
});

Credit: @rolando-benjamin-vaz-ferreira

Executing a callback function after .forEach finishes

This is a case where the more modern approach is to use promises

function asyncFunction(item) {   // return a promise  return new Promise((resolve, reject) => {    setTimeout(() => {      console.log("in async function, item is " + item)      // resolve promise when data is ready      resolve(item)    }, Math.random()*2000)// random delay to stagger order of returns
  })
}
// create array of promiseslet promiseArray = [1, 2, 3].map(asyncFunction);
// runs when all promises are resolvedPromise.all(promiseArray).then(results => {  console.log('all done')  // results array will be in same order as original array  console.log('results are: ', results)})

.as-console-wrapper {max-height: 100%!important;top:0}

Javascript - Callback after all nested forEach loops are completed

Using forEach's 2nd parameter, the index, you can carry out a check whether all loops are done each time you run the innermost loop.

Thus with only a few lines added to your code you get this:

const scanFiles = function(accounts, cb) {
    let dirs = ['pending', 'done', 'failed'];
    let jobs = [];

    accounts.forEach(function(account, accIndex) {
        dirs.forEach(function(dir, dirIndex) {
            fs.readdir(account + '/' + dir, function(err, files) {
                files.forEach(function(file, fileIndex) {
                    //do something
                    //add file to jobs array
                    jobs.push(file);

                    // Check whether each loop is on its last iteration
                    const filesDone = fileIndex >= files.length - 1;
                    const dirsDone = dirIndex >= dirs.length - 1;
                    const accsDone = accIndex >= accounts.length - 1;

                    // all three need to be true before we can run the callback
                    if (filesDone && dirsDone && accsDone) {
                        cb(jobs);
                    }
                });
            });
        });
    });
}

run a callback function after forEach loop finish

Another approach using async-await to avoid callback hell.

const usersProfile = async () => {
  let interval = 1000;

  const delayPromise = (data, delayDuration) => {
    return new Promise((resolve) => {
      setTimeout(() => {
        if(data.status === 'online'){
          console.log(`${data.name} ${data.items} ${data.profile} ${data.images}`);
          resolve();
        }
      }, delayDuration)
    });
  };

  try {
    const userData = await db.select('*').from('users').returning('*');
    const promises = userData.map((data, index) => delayPromise(data, index * interval));
    await Promise.all(promises);
    // here you can write your effects
  } catch (e) {
    console.log(e);
  }
}

How to wait for forEach to complete with asynchronous callbacks?

Iterable.forEach, Map.forEach, and Stream.forEach are meant to execute some code on each element of a collection for side effects. They take callbacks that have a void return type. Consequently, those .forEach methods cannot use any values returned by the callbacks, including returned Futures. If you supply a function that returns a Future, that Future will be lost, and you will not be able to be notified when it completes. You therefore cannot wait for each iteration to complete, nor can you wait for all iterations to complete.

Do NOT use .forEach with asynchronous callbacks.

Instead, if you want to wait for each asynchronous callback sequentially, just use a normal for loop:

for (var mapEntry in gg.entries) {
  await Future.delayed(const Duration(seconds: 5));
}

(In general, I recommend using normal for loops over .forEach in all but special circumstances. Effective Dart has a mostly similar recommendation.)

If you really prefer using .forEach syntax and want to wait for each Future in succession, you could use Future.forEach (which does expect callbacks that return Futures):

await Future.forEach(
  gg.entries,
  (entry) => Future.delayed(const Duration(seconds: 5)),
);

If you want to allow your asynchronous callbacks to run concurrently (and possibly in parallel), you can use Future.wait:

await Future.wait([
  for (var mapEntry in gg.entries)
    Future.delayed(const Duration(seconds: 5)),
]);

See https://github.com/dart-lang/linter/issues/891 for a request for an analyzer warning if attempting to use an asynchronous function as a Map.forEach or Iterable.forEach callback (and for a list of many similar StackOverflow questions).

JavaScript - Run callback after all fetch requests to finish in multiple for loops - errored or not

Push all of your promises into an array, and then use Promise.allSettled(promises).then((results) => {})

Documentation: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/allSettled

Example:

const promises = images.map(async (image) => {
    // do some async work
    return fetch(whatever); // don't .catch this

})

Promise.allSettled(promises).then((results) => {
  // results is a result of errors or successes
})

Async function and callbacks inside a forEach loop

Your questions

toReturn.push(doc) doesn't work and on the last callback an empty array is returned. Why is that?

You're calling callback before MongooseModel.AsyncFunction gets any chance to execute your function(err,doc).

If there's an error, or !docs is true, return callback('Error') is never called. return callback(null, toReturn) is always called.

The successful return is called immediately after scheduling your async functions. It's still possible that the callback('Error') will be called sometime later by MongooseModel.AsyncFunction.

the script crashes because multiple callbacks have been called

That's exactly the behaviour you asked for! Your second code literally says: "For each element, call the callback once".

---

Some pointers

All in all, I think the single thing you have wrong is that calling the callback somewhat resembles returning from the function.

That's not the case! Your function Example is supposed to schedule some async things to happen, then return immediately (returning nothing) but promising that either callback(error) or callback(null, success) will be called sometime later in the future.

Therefore, it doesn't make sense to ever say return callback() - just callback() will do. By the time that you have some data to call callback with, the function Example will already have finished executing. The function that eventually calls callback will the an anonymous function passed to MongooseModel.AsyncFunction, not Example itself.

---

Now what?

You could try an approach like this:

function Example(array, callback){
    var toReturn = [];
    var previousError = false;

    array.forEach(function(el){
        MongooseModel.AsyncFunction(el, function(err, doc){
            if (previousError) {
                return;
            }
            if(err || !doc) {
                previousError = true;
                callback('Error');
            }
            toReturn.push(doc)
            if (toReturn.length === array.length) {
                // that was the last push - we have completed
                callback(null, toReturn);
            }
        });
    });
}

Stuff that happens here:

• Every time your AsyncFunction completes, you aggregate the result. If that was the last one, you can finally call callback and pass the whole data. (Otherwise, we're waiting for a few more functions, so no callback)

• If there was an error somewhere along the way, we report it immediately, but also take a note that the error has been reported already, so that further executions of AsyncFunction that have been scheduled don't do anything in particular. This prevents your callback from potentially being called twice or more.

• Be careful though - the order of elements inside toReturn will be random, depending on which async tasks complete first.

---

But this is messy

Oh yes. That's why we don't really do callbacks anymore. There's a pattern called promises that makes it significantly easier to work with async callback spaghetti, I suggest to read up a little on this topic before moving forward.

• https://www.promisejs.org/

Let's make it neat

Using promises, this kind of code would look something like:

function Example(array) {
    var promises = array.map(function(el) {
        return MongooseModel.AsyncFunction(el);
    });
    return Promise.all(promises);
}

which is much shorter and also doesn't have any problems with error handling or ordering of output items like our previous example.

Usage is simple - instead of this

Example([1, 2, 3], function(error, results) {
    if (error !== null) {
        // ...
    } else { 
        // ...
    }
})

you call like this:

Example([1, 2, 3]).then(
    function(results) {
        // ...
    },
    function(error) {
        // ...
    }
);

Oh, this relies on MongooseModel.AsyncFunction returning a promise on itself, but Mongoose already does this and so do most libraries. If a library doesn't, you can easily add the promise support (see NodeRedis for example).

How to do something with each result of AsyncFunction before returning it?

Easy too!

function Example(array) {
    var promises = array.map(function(el) {
        return MongooseModel.AsyncFunction(el).then(function(doc) {
            return doc + "some modifications";
        });
    });
    return Promise.all(promises);
}

Or if you need additional error handling:

function Example(array) {
    var promises = array.map(function(el) {
        return MongooseModel.AsyncFunction(el).then(function(doc) {
            if (theresSomethingWrongWith(doc) {
                return Promise.reject(new Error("ouch!"));
            }
            return doc + "some modifications";
        });
    });
    return Promise.all(promises);
}

Think of returning a rejected promise as something similar to raising an exception - it will naturally bubble up all the way to the returning promise of Example.

Run callback function after forEach is done

What you want to do is to make all your asynchronous processes converge into a single promise that you can use to execute the callback at the correct moment.

Lets start at the point each process is complete, which I assume is in the callback passed to the mv() function in unzipData(). You want to wrap each of these asynchronous actions in a Promise that resolves in the callback and you also want to use these promises later and for that you use the .map() method to collect the promises in an array (instead of .forEach()).

Here's the code:

var promises = filelist.map((filePath) => {
  if (!filePath.endsWith('.zip')) {
    var components = filePath.split('/');
    var component = components[components.length-1];
    return new Promise((resolve, reject) =>
      mv(filePath, datadir + `processing/${uid}/${component}`, (err) => {
        if(err) {
          console.log('move file err: ');
          reject(); // Or resolve() if you want to ignore the error and not cause it to prevent the callback from executing later
        } else {
          console.log('move file done');
          resolve();
        }
      }));
  }
  return Promise.resolve();
});

(if the asynchronous action is not to be executed, a Promise that resolves immediately is returned instead)

Now, we can turn this list of Promises into a single Promise that resolves when all of the promises in the list has resolved:

var allPromise = Promise.all(promises);

Next, we need to look further up in the code. We can see that the code we've just been looking at is itself part of an event handler of an asynchronous action, i.e. fs.createReadStream(). You need to wrap that in a promise that gets resolved when the inner promises resolve and this is the promise that the unzipData() function shall return:

function unzipData(path, fileName, uid) {
  console.log('get into unzip');
  console.log('creating: ', path + fileName);
  return new Promise((outerResolve) =>
    fs.createReadStream(path + fileName)
    .pipe(unzip.Extract({path: path}))
    .on('close', () => {
      console.log('get into unzip close');
      var filelist = listFile(path);

      // Code from previous examples

      allPromise.then(outerResolve);
    }));
}

Next, we look at the functions that use unzipData(): getDataReg() and getDataFtp(). They only perform one asynchronous action so all you need to do is to make them return a promise that resolves when the promise returned by unzipData() resolves.

Simplified example:

exports.getDataReg = (url, uid) => {
  return new Promise((resolve, reject) => {

    // ...

    https.get(url, (response) => {
      response.pipe(file);
      file.on('finish', () => {
        unzipData(datadir + `download/${uid}/`, fileName, uid)
          .then(resolve);
      });
    }).on('error', (err) => { // Handle errors
      fs.unlink(datadir + `download/${uid}/${fileName}`);
      reject(); // Or resolve() if you want to ignore the error and not cause it to prevent the callback from executing later
    });

    // ...

  });
}

Finally, we get to the processArray() function and here you need to do the same thing we did to begin with: map the processes into a list of promises. First, the process function passed needs to return the promises returned by getDataReg() and getDataFtp():

// Second param, download and post process
(url) => {
  if(url.startsWith('http')) {
    return getDataReg(url, uid);
  }
  else if(url.startsWith('ftp')) {
    return getDataFtp(url, uid);
  }
  else {
    console.log('not a valid resource');
  }
  return Promise.reject(); // or Promise.resolve() if you want invalid resources to be ignored and not prevent the callback from executing later
}

Now, your processArray() function can look like this:

exports.processArray = (items, process, callback) =>
  Promise.all(items.map(process))
    .then(callback)
    .catch(() => console.log('Something went wrong somewhere'));

Your callback will get invoked when all asynchronous actions have completed, regardless of in which order they do. If any one of the promises rejects, the callback will never be executed so manage your promise rejections accordingly.

Here's a JSFiddle with the complete code: https://jsfiddle.net/upn4yqsw/

---

Related Topics