Building Scalable Applications with Node.js and MongoDB

Building applications that can handle growth is crucial. This guide covers strategies to make your Node.js and MongoDB stack highly scalable.

Understanding Scalability

Scalability is the ability of a system to handle increased load by adding resources (horizontal scaling) or upgrading existing resources (vertical scaling).

Node.js Cluster Module

const cluster = require('cluster');
const os = require('os');
const express = require('express');

if (cluster.isMaster) {
    const numCPUs = os.cpus().length;
    console.log("Master process.pid setting up numCPUs workers");
    
    for (let i = 0; i < numCPUs; i++) {
        cluster.fork();
    }
    
    cluster.on('exit', (worker) => {
        console.log("Worker worker.process.pid died.Forking new one...");
        cluster.fork();
    });
} else {
    const app = express();
    const PORT = process.env.PORT || 3000;
    
    app.get('/', (req, res) => {
        res.send("Handled by worker 20004");
    });
    
    app.listen(PORT, () => {
        console.log("Worker process.pid listening on port PORT");
    });
}

Load Balancing with PM2

# Install PM2 globally
npm install -g pm2

# Start app in cluster mode
pm2 start app.js -i max

# Monitor processes
pm2 monit

# Scale up/down dynamically
pm2 scale app 8

# Enable process auto-restart
pm2 start app.js --watch

# Save configuration for system startup
pm2 save
pm2 startup

MongoDB Indexing for Query Performance

// Creating efficient indexes
db.users.createIndex({ email: 1 });                  // Single field
db.users.createIndex({ name: 1, age: -1 });          // Compound
db.users.createIndex({ description: "text" });       // Text search
db.orders.createIndex({ createdAt: 1 }, { expireAfterSeconds: 2592000 }); // TTL

// Covered queries (all fields in index)
db.users.createIndex({ name: 1, email: 1, age: 1 });
db.users.find({ name: "Mohamed" }, { name: 1, email: 1, age: 1, _id: 0 }); // Index only

// Analyze query performance
db.users.find({ age: { $gt: 25 } }).explain("executionStats");

Database Sharding Strategy

// Enable sharding on database
sh.enableSharding("myAppDB");

// Choose shard key
sh.shardCollection("myAppDB.users", { userId: "hashed" });  // Hashed sharding for even distribution
// Or range-based sharding
sh.shardCollection("myAppDB.events", { createdAt: 1 });

// Add shards
sh.addShard("shard1/localhost:27018");
sh.addShard("shard2/localhost:27019");
sh.addShard("shard3/localhost:27020");

Caching with Redis

const redis = require('redis');
const client = redis.createClient();

// Cache middleware
const cacheMiddleware = (duration = 60) => {
    return async (req, res, next) => {
        const key = "cache: req.originalUrl";
        
        try {
            const cachedData = await client.get(key);
            
            if (cachedData) {
                return res.json(JSON.parse(cachedData));
            }
            
            // Store original json method
            const originalJson = res.json;
            
            // Override json method to cache response
            res.json = function(data) {
                // Cache the response
                client.setex(key, duration, JSON.stringify(data));
                
                // Call original json method
                return originalJson.call(this, data);
            };
            
            next();
        } catch (err) {
            next();
        }
    };
};

// Usage
app.get('/api/users', cacheMiddleware(300), async (req, res) => {
    const users = await User.find();
    res.json(users);
});

Message Queues with RabbitMQ/Bull

const Queue = require('bull');
const emailQueue = new Queue('email sending', 'redis://127.0.0.1:6379');

// Add job to queue
emailQueue.add({
    to: 'user@example.com',
    subject: 'Welcome',
    template: 'welcome-email'
});

// Process jobs
emailQueue.process(async (job) => {
    const { to, subject, template } = job.data;
    await sendEmail(to, subject, template);
    return { success: true };
});

// Handle failed jobs
emailQueue.on('failed', (job, err) => {
    console.error("Job job.id failed: ", err);
    // Implement retry logic
});

Database Connection Pooling

// MongoDB connection with mongoose
const mongoose = require('mongoose');

const connectDB = async () => {
    try {
        const conn = await mongoose.connect(process.env.MONGO_URI, {
            maxPoolSize: 100,          // Maximum number of connections
            minPoolSize: 10,           // Minimum number of connections
            maxIdleTimeMS: 30000,      // Close idle connections after 30s
            socketTimeoutMS: 45000,    // Close sockets after 45s
            family: 4                  // Use IPv4
        });
        
        console.log("MongoDB Connected: conn.connection.host");
        
        // Monitor connection pool
        setInterval(() => {
            const poolSize = mongoose.connection.client.topology?.s.pool?.size();
            console.log("Connection pool size: poolSize");
        }, 60000);
    } catch (error) {
        console.error('Database connection error:', error);
        process.exit(1);
    }
};

Rate Limiting and Throttling

const rateLimit = require('express-rate-limit');
const RedisStore = require('rate-limit-redis');

// Distributed rate limiting with Redis
const limiter = rateLimit({
    store: new RedisStore({
        client: redisClient,
        prefix: 'rate-limit:'
    }),
    windowMs: 15 * 60 * 1000, // 15 minutes
    max: 100,                  // limit each IP to 100 requests per windowMs
    message: 'Too many requests, please try again later.',
    standardHeaders: true,
    legacyHeaders: false,
    keyGenerator: (req) => {
        // Use user ID if authenticated, otherwise IP
        return req.user?.id || req.ip;
    }
});

// Apply to all API routes
app.use('/api/', limiter);

// Stricter limits for sensitive endpoints
const authLimiter = rateLimit({
    windowMs: 15 * 60 * 1000,
    max: 5, // 5 failed attempts per 15 minutes
    skipSuccessfulRequests: true
});
app.use('/api/auth/login', authLimiter);

Horizontal Scaling with Load Balancer (Nginx)

# nginx.conf
upstream node_app {
    least_conn;  # Distribute to server with least connections
    server 127.0.0.1:3001;
    server 127.0.0.1:3002;
    server 127.0.0.1:3003;
    keepalive 32;
}

server {
    listen 80;
    server_name myapp.com;
    
    location / {
        proxy_pass http://node_app;
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection 'upgrade';
        proxy_set_header Host $host;
        proxy_cache_bypass $http_upgrade;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
    }
}

Database Optimization Techniques

• Denormalization: Duplicate data to avoid joins
• Aggregation pipelines: Process data within MongoDB
• Read preferences: Distribute reads to secondary replicas
• TTL indexes: Auto-delete old data
• Partial indexes: Index only relevant documents

Monitoring and Observability

// Winston logging with Elasticsearch
const winston = require('winston');
const { ElasticsearchTransport } = require('winston-elasticsearch');

const logger = winston.createLogger({
    level: 'info',
    format: winston.format.json(),
    transports: [
        new winston.transports.File({ filename: 'error.log', level: 'error' }),
        new winston.transports.File({ filename: 'combined.log' }),
        new ElasticsearchTransport({
            level: 'info',
            clientOpts: { node: 'http://localhost:9200' },
            index: 'app-logs'
        })
    ]
});

// Metrics with Prometheus
const promClient = require('prom-client');
const httpRequestDuration = new promClient.Histogram({
    name: 'http_request_duration_ms',
    help: 'Duration of HTTP requests in ms',
    labelNames: ['method', 'route', 'status_code']
});

app.use((req, res, next) => {
    const start = Date.now();
    res.on('finish', () => {
        const duration = Date.now() - start;
        httpRequestDuration.labels(req.method, req.route?.path, res.statusCode).observe(duration);
    });
    next();
});

Conclusion

Scalability requires careful planning across multiple layers: application, database, caching, and infrastructure. Start with a solid foundation, measure performance, identify bottlenecks, and scale only what's needed.