Core Optimization Techniques

When dealing with millions of records, API response times can degrade significantly. Implementing a combination of pagination, indexing, caching, and asynchronous processing can dramatically improve performance and reduce server load.

1. Implementing Pagination

Pagination limits the data returned per request, reducing both database query time and network transfer overhead. Instead of retrieving all records, the server returns manageable chunks.

const fastify = require('fastify')({ logger: true });

fastify.get('/records', async (request, reply) => {
    const { pageNum = 1, limit = 20 } = request.query;
    const offset = (pageNum - 1) * limit;
    
    const results = await fetchPaginatedRecords(offset, limit);
    return { 
        data: results.rows,
        meta: { 
            currentPage: pageNum,
            totalRecords: results.count 
        }
    };
});

async function fetchPaginatedRecords(offset, limit) {
    // Implementation using your ORM or query builder
    // Example: SELECT * FROM records LIMIT ? OFFSET ?
}

fastify.listen({ port: 3000 });

2. Database Index Optimization

Proper indexing on frequently queried columns accelerates data retrieval. Analyze query patterns and create composite indexes for multi-column filters.

// Using Knex.js for index creation in migrations
exports.up = function(knex) {
    return knex.schema.table('users', function(table) {
        table.index(['status', 'created_at'], 'idx_status_created');
        table.index('email', 'idx_email_unique');
    });
};

// Query benefiting from the index
const activeRecentUsers = await knex('users')
    .where('status', 'active')
    .orderBy('created_at', 'desc')
    .limit(50);

3. Redis Caching Layer

Caching frequently accessed data reduces database hits. Redis provides fast in-memory storage for hot data with configurable expiration.

const Redis = require('ioredis');
const redis = new Redis();

async function getCachedData(cacheKey, fetchFn, ttlSeconds = 3600) {
    const cached = await redis.get(cacheKey);
    
    if (cached) {
        return JSON.parse(cached);
    }
    
    const freshData = await fetchFn();
    await redis.setex(cacheKey, ttlSeconds, JSON.stringify(freshData));
    return freshData;
}

// Usage in route handler
fastify.get('/products', async (req, res) => {
    return await getCachedData(
        'products:featured',
        () => fetchFeaturedProducts(),
        1800
    );
});

4. Background Processing with Message Queues

Offload intensive operations to background workers using message queues like Bull or RabbitMQ. This keeps API endpoints responsive.

const Queue = require('bull');
const reportQueue = new Queue('report-generation');

// Endpoint triggers background job
fastify.post('/reports/generate', async (req, res) => {
    const job = await reportQueue.add({
        userId: req.user.id,
        filters: req.body.filters
    });
    
    return { jobId: job.id, status: 'processing' };
});

// Worker process
reportQueue.process(async (job) => {
    const data = await generateLargeReport(job.data.filters);
    await storeReport(job.data.userId, data);
    return { reportId: data.id };
});

5. Response Compression

Compress API responses to reduce payload size and transfer time. Gzip or Brotli compression significantly reduces bandwidth usage for text-heavy responses.

const fastify = require('fastify')();
const fastifyCompress = require('@fastify/compress');

fastify.register(fastifyCompress, {
    global: true,
    encodings: ['br', 'gzip', 'deflate']
});

// All responses now automatically compressed
fastify.get('/large-dataset', async (req, res) => {
    return await fetchLargeDataset();
});

6. Connection Pooling

Database connection pools reuse existing connections, eliminating connection overhead for each request.

const { Pool } = require('pg');

const dbPool = new Pool({
    max: 20,
    idleTimeoutMillis: 30000,
    connectionTimeoutMillis: 2000
});

async function queryDatabase(sql, params) {
    const client = await dbPool.connect();
    try {
        return await client.query(sql, params);
    } finally {
        client.release();
    }
}

7. Streaming Large Datasets

For exports or large data transfers, use streams to avoid loading entire datasets into memory.

const { pipeline } = require('stream/promises');
const { Readable } = require('stream');

fastify.get('/export/csv', async (req, res) => {
    res.header('Content-Type', 'text/csv');
    res.header('Content-Disposition', 'attachment; filename=export.csv');
    
    const dbStream = await createDatabaseStream();
    const csvTransform = createCsvTransformer();
    
    await pipeline(dbStream, csvTransform, res.raw);
});

function createDatabaseStream() {
    return new Readable({
        async read() {
            // Batch read from database
        }
    });
}

8. Cache Warming on Startup

Preload frequently accessed data into cache during application initialization to avoid cold-start delays.

async function warmCache() {
    const hotData = await Promise.all([
        fetchSystemConfig(),
        fetchPopularCategories(),
        fetchActivePromotions()
    ]);
    
    await redis.mset(
        'sys:config', JSON.stringify(hotData[0]),
        'categories:popular', JSON.stringify(hotData[1]),
        'promotions:active', JSON.stringify(hotData[2])
    );
}

// Execute during startup
fastify.addHook('onReady', warmCache);

9. HTTP Caching Headers

Implement proper cache-control headers to allow client-side and proxy caching.

fastify.get('/stable-data', async (req, res) => {
    res.header('Cache-Control', 'public, max-age=3600, must-revalidate');
    res.header('ETag', generateETag(data));
    
    // Check If-None-Match for 304 response
    if (req.headers['if-none-match'] === cachedETag) {
        res.code(304).send();
        return;
    }
    
    return data;
});

10. Cluster Mode for CPU Utilization

Leverage multi-core systems by running multiple Node.js instances behind a cluster manager.

const cluster = require('cluster');
const os = require('os');

if (cluster.isPrimary) {
    const cpuCount = os.cpus().length;
    
    for (let i = 0; i < cpuCount; i++) {
        cluster.fork();
    }
    
    cluster.on('exit', (worker) => {
        console.log(`Worker ${worker.id} restarting...`);
        cluster.fork();
    });
} else {
    // Worker process runs the server
    require('./server');
}

11. Request Monitoring and Logging

Implement comprehensive logging to identify slow endpoints and track performance metrics.

const pino = require('pino');
const logger = pino({ level: 'info' });

fastify.addHook('onRequest', async (req, res) => {
    req.startTime = Date.now();
});

fastify.addHook('onResponse', async (req, res) => {
    const duration = Date.now() - req.startTime;
    logger.info({
        method: req.method,
        url: req.url,
        statusCode: res.statusCode,
        responseTime: duration
    });
});

12. Load Balancing Architecture

Distribute incoming traffic across multiple server instances using a load balancer for horizontal scaling.

// Nginx configuration example
// upstream api_servers {
//     server api1.example.com:3000;
//     server api2.example.com:3000;
//     server api3.example.com:3000;
// }
// 
// server {
//     location /api/ {
//         proxy_pass http://api_servers;
//         proxy_set_header Host $host;
//     }
// }