Connection pool is closed using Mongoose

Question

I have a little web app which serves data from a Mongo database. I have configured 2 cron jobs (through Heroku scheduler) to run everyday and manipulate a remote db. The problem is that I need those jobs to conclude and not keep running after finishing, that means closing my connection, otherwhise my function keeps running.

When I call mongoose.disconnect() in one of my files I get Mongo error: Connection pool closed.

This is the problematic file:

require('dotenv').config();
const malagaCulturaScrapper = require("../scrappers/malagaCultura");
const Event = require("../schemas/eventSchema");
const mongoose = require("mongoose");

(async () => {
    try {
        await mongoose.connect((process.env.DEV_MODE === "true") ? process.env.SAMPLEDB_URL:process.env.DATABASE_URL, {
            useNewUrlParser: true,
            useUnifiedTopology: true
        })
    } catch (err) {
        console.log('Error: ' + err)
    }
})()

const scrapeAndSave = async() =>{
    await Event.deleteMany({}).exec();
    await malagaCulturaScrapper.malagaCulturaScrapper();
    mongoose.disconnect();
}

scrapeAndSave();

Which calls this function:

const malagaCulturaScrapper = async() =>{
        await scrapeIt("https://malagadecultura.com/agenda/", {
            events: {
                listItem: "article",
                data: {
                    title: "h4",
                    time: ".mec-event-time",
                    date: {
                        closest: "li",
                        attr: "id"                        
                    },
                    event_img: {
                        selector: ".attachment-thumbnail",
                        attr: "src"
                    },
                    event_link:{
                        selector: "h4 a",
                        attr: "href",
                    },
                    location: ".mec-event-loc-place",
                }
        }
        }).then(({ data, response }) => {

            for(const index in data.events){
                let newEvent = new Event({
                    _id: new mongoose.Types.ObjectId(),
                    title: data.events[index].title,
                    dateTime: createDateObject(data.events[index].date.substring(59), data.events[index].time),
                    event_img: data.events[index].event_img,
                    event_link: data.events[index].event_link,
                    location: data.events[index].location,
                })
                Event.addEvents(newEvent);
            }
        })

}

And this is my adding function on the schema:

Event.addEvents = async(newEvent) => {
    await Event.findOne({
        location: newEvent.location || {
            '$regex': newEvent.location, $options: 'i',
        },
        dateTime: { $in: newEvent.dateTime }
    }, (err, event) => {
        if(err) throw err;
        if (!event && !helpers.isPastEvent(newEvent.dateTime[0], Date.now()) && new Date(newEvent.dateTime[0]).getFullYear() === new Date(Date.now()).getFullYear()) {
            newEvent.save((err) => {
                if (err) throw err;
            })
        } 
    })
}

Lastly, the stacktrace I get when running the file:

/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/helpers/promiseOrCallback.js:19
            throw error;
            ^

MongoError: connection pool closed
    at ConnectionPool.close (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection_pool.js:300:34)
    at Server.destroy (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/server.js:213:17)
    at destroyServer (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:806:10)
    at eachAsync (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:353:25)
    at eachAsync (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/utils.js:131:5)
    at s.sessionPool.endAllPooledSessions (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:351:7)
    at topology.endSessions (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sessions.js:592:13)
    at command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:501:45)
    at cb (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/topology.js:681:26)
    at fn (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection_pool.js:350:13)
    at Object.handleOperationResult [as cb] (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/server.js:558:5)
    at Connection.write (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection.js:378:26)
    at _command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/wireprotocol/command.js:120:10)
    at Object.command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/wireprotocol/command.js:28:5)
    at Connection.command (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/cmap/connection.js:171:8)
    at s.pool.withConnection (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongodb/lib/core/sdam/server.js:285:12)
Emitted 'error' event at:
    at /home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/model.js:4844:13
    at /home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/helpers/promiseOrCallback.js:16:11
    at /home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/model.js:4865:21
    at (anonymous function).call (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/query.js:4411:18)
    at Immediate.Query.base.findOne.call (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mongoose/lib/query.js:2145:7)
    at Immediate.<anonymous> (/home/roma/Escritorio/Programming-projects/mlg-events/node_modules/mquery/lib/utils.js:116:16)
    at runCallback (timers.js:705:18)
    at tryOnImmediate (timers.js:676:5)
    at processImmediate (timers.js:658:5)

I would really appreciate any guidance or help on the subject. Thanks in advance.

Answer 1

The code is trying to disconnect before the queries have completed.

Use the Promise API to mongoose by not supplying any callback functions and make sure you await promises so the disconnect run's after everything has completed.

const malagaCulturaScrapper = async() =>{
    const { data, response } = await scrapeIt("https://malagadecultura.com/agenda/", {
         ...eventsInfo
    })
    for(const index in data.events){
        let newEvent = new Event({
            _id: new mongoose.Types.ObjectId(),
            title: data.events[index].title,
            dateTime: createDateObject(data.events[index].date.substring(59), data.events[index].time),
            event_img: data.events[index].event_img,
            event_link: data.events[index].event_link,
            location: data.events[index].location,
        })
        await Event.addEvents(newEvent);
    }
}

Event.addEvents = async(newEvent) => {
    const event = await Event.findOne({
        location: newEvent.location || {
            '$regex': newEvent.location, $options: 'i',
        },
        dateTime: { $in: newEvent.dateTime }
    })

    if (!event && !helpers.isPastEvent(newEvent.dateTime[0], Date.now()) && new Date(newEvent.dateTime[0]).getFullYear() === new Date(Date.now()).getFullYear()) {
        await newEvent.save()
    })
}