youtube-dl in a javascript script errors

Question

i have youtube-dl in a javascript script ,trying to download hundreds of caption files. I am getting errors

I have a javascript script. I am trying to download ~800 caption files using youtube-dl, i'm getting time out errors and it isn't downloading the files. It seems like it is moving too fast for my computer. I need help pausing the task until the download finishes and then starting the next one.

It is working very less data in videos array.

var json2csv = require('json2csv');
var fs = require('fs');
var youtubedl = require('youtube-dl');
// var fields = ["youtube_id", "title", "date", "duration", "captioned", "views"];
var videos = [
// More objects ~800+
];
 for (i = 0; i < videos.length; i++) { 
 var v = videos[i];
 var url = 'https://youtu.be/';
 var options = {};
 if (v["captioned"] == 'No') { 
 var url = url + v["youtube_id"];
 console.log(url);
 var options = {
 auto: true,
 all: false,
 lang: 'en',
 cwd: __dirname + "/auto_generated_captions",
 };
 youtubedl.getSubs(url, options, function(err, files) {
 console.log("did i get here?");
 if (err) throw err;
 console.log('subtitle files downloaded:', files);
 });
 };
 }; 

Answer 1

You are right. You download too much data at the same time. Try to control the concurrency flow with promise library like bluebird:

var json2csv = require('json2csv');
var fs = require('fs');
var youtubedl = require('youtube-dl');
var promise = require('bluebird');
// var fields = ["youtube_id", "title", "date", "duration", "captioned", "views"];
var videos = [
 // More objects ~800+
];
promise
 .map(videos, function (v) {
 var url = 'https://youtu.be/';
 var options = {};
 if (v["captioned"] == 'No') {
 var url = url + v["youtube_id"];
 console.log(url);
 var options = {
 auto: true,
 all: false,
 lang: 'en',
 cwd: __dirname + "/auto_generated_captions",
 };
 return new Promise(function (resolve, reject) {
 youtubedl.getSubs(url, options, function (err, files) {
 console.log("did i get here?");
 if (err) {
 reject(err);
 } else {
 console.log('subtitle files downloaded:', files);
 resolve(files);
 }
 });
 });
 } else {
 // return a promise for this case
 }
 }, { concurrency: 5 })
 .then(function (results) {
 console.log(results);
 });