Pourquoi mon code utilisant insertMany() ignore certains enregistrements et insère plusieurs fois les mêmes enregistrements?

Question

J'ai 9577 enregistrements uniques dans un fichier csv.

Ce code insère 9800 enregistrements et insère pas tous les enregistrements, mais des doublons de certains d'entre eux. Une idée pour laquelle il n'insère pas les enregistrements 9577 uniques et aussi les doublons de certains d'entre eux? Ci-dessous, insérer également continuer à faire partie du code afin que vous obtenez l'image entière

function bulkImportToMongo(arrayToImport, mongooseModel) { 
    const Model = require(`../../../models/${mongooseModel}`); 
    let batchCount = Math.ceil(arrayToImport.length/100); 
    console.log(arrayToImport.length); 
    let ops = []; 

    for (let i = 0; i < batchCount; i++) { 
    // console.log(i); 
    let batch = arrayToImport.slice(i, i + 100); 
    console.log(batch.length); 
    ops.push(Model.insertMany(batch)); 
    } 
    return ops; 

    return Promise.all(ops).then(results => { 
    // results is an array of results for each batch 
    console.log("results: ", results); 
    }); 
} 

Je Parse le fichier csv

const Promise = require("bluebird"); 
const csv = require("fast-csv"); 
const path = require("path"); 
const fs = Promise.promisifyAll(require("fs")); 

const promiseCSV = Promise.method((filePath, options) => { 
    return new Promise((resolve, reject) => { 
    var records = []; 
    csv 
     .fromPath(filePath, options) 
     .on("data", record => { 
     records.push(record); 
     }) 
     .on("end",() => { 
     // console.log(records); 
     resolve(records); 
     }); 
    }); 
}); 

Et le script ici qui relie tous ensemble:

const path = require("path"); 
const promiseCSV = require("./helpers/ImportCSVFiles"); 
const { 
    connectToMongo, 
    bulkImportToMongo 
} = require("./helpers/mongoOperations"); 

const filePath = path.join(__dirname, "../../data/parts.csv"); 
const options = { 
    delimiter: ";", 
    noheader: true, 
    headers: [ 
    "facility", 
    "partNumber", 
    "partName", 
    "partDescription", 
    "netWeight", 
    "customsTariff" 
    ] 
}; 

connectToMongo("autoMDM"); 
promiseCSV(filePath, options).then(records => { 
    bulkImportToMongo(records, "parts.js"); 
}); 

Answer 1

// Il semble que votre problème soit simplement i ++. Peut-être que vous vouliez dire i + = 100?

for (let i = 0; i < batchCount; i+=100 /* NOT i++ */) { 
    //... 
} 

Answer 2

Je l'ai résolu.

J'espère que cela aide les autres ... :-)

J'ai eu deux erreurs, dans la fonction promiseCSV (changé à parseCSV) et seconde j'avais mauvaise logique bulkImportToMongo.

Solution complète:

J'analysé et importé 602.198 objets et voici combien de temps il a fallu à l'aide noeud --max_old_space_size = 8000 sur un MacBook Pro avec 8 Go de RAM.

console

➜ database git:(master) ✗ node --max_old_space_size=8000 partImport.js 
Connected to db! 
Time to parse file: : 5209.325ms 
Disconnected from db! 
Time to import parsed objects to db: : 153606.545ms 
➜ database git:(master) ✗ 

parseCSV.js

const csv = require("fast-csv"); 

function promiseCSV(filePath, options) { 
    return new Promise((resolve, reject) => { 
    console.time("Time to parse file"); 
    var records = []; 
    csv 
     .fromPath(filePath, options) 
     .on("data", record => { 
     records.push(record); 
     }) 
     .on("end",() => { 
     console.timeEnd("Time to parse file"); 
     resolve(records); 
     }); 
    }); 
} 

module.exports = promiseCSV; 

mongodb.js

const mongoose = require("mongoose"); 
mongoose.Promise = global.Promise; 

function connectToMongo(databaseName) { 
    mongoose.connect(`mongodb://localhost:27017/${databaseName}`, { 
    keepAlive: true, 
    reconnectTries: Number.MAX_VALUE, 
    useMongoClient: true 
    }); 
    console.log("Connected to db!"); 
} 

function disconnectFromMongo() { 
    mongoose.disconnect(); 
    console.log("Disconnected from db!"); 
} 

function bulkImportToMongo(arrayToImport, mongooseModel) { 
    const Model = require(`../../../models/${mongooseModel}`); 
    const batchSize = 100; 
    let batchCount = Math.ceil(arrayToImport.length/batchSize); 
    let recordsLeft = arrayToImport.length; 
    let ops = []; 
    let counter = 0; 
    for (let i = 0; i < batchCount; i++) { 
    let batch = arrayToImport.slice(counter, counter + batchSize); 
    counter += batchSize; 
    ops.push(Model.insertMany(batch)); 
    } 
    return Promise.all(ops); 
} 

module.exports.bulkImportToMongo = bulkImportToMongo; 
module.exports.connectToMongo = connectToMongo; 
module.exports.disconnectFromMongo = disconnectFromMongo; 

partImport.js

const path = require("path"); 
const parseCSV = require("./helpers/parseCSV"); 
const { 
    connectToMongo, 
    disconnectFromMongo, 
    bulkImportToMongo 
} = require("./helpers/mongodb"); 

const filePath = path.join(__dirname, "../../data/parts.csv"); 
const options = { 
    delimiter: ";", 
    noheader: true, 
    headers: [ 
    "facility", 
    "partNumber", 
    "partName", 
    "partDescription", 
    "netWeight", 
    "customsTariff" 
    ] 
}; 

connectToMongo("autoMDM"); 
parseCSV(filePath, options) 
    .then(records => { 
    console.time("Time to import parsed objects to db"); 
    return bulkImportToMongo(records, "parts.js"); 
    }) 
    /* .then(result => 
    console.log("Total batches inserted: ", result, result.length) 
) */ 
    .then(() => { 
    disconnectFromMongo(); 
    console.timeEnd("Time to import parsed objects to db"); 
    }) 
    .catch(error => console.log(error));