Paging in the Database

Part of the Databases with MongoDB section of Coddy's Node.js journey. Lesson 27 of 29.

A shelf of ten thousand books cannot go out in one response. The client asks for one page at a time, and the database should send only that page.

skip(n) passes over the first n matches, and limit(n) keeps the next n. Page 1 starts at the beginning, so page p skips (p - 1) * size. Send the total with the page, counted with the same filter, so the client knows how many pages there are.

// page 3 of the error log, 20 entries a page
const entries = await log
    .find({ level: 'error' })
    .skip((3 - 1) * 20)
    .limit(20)
    .toArray();
const total = await log.countDocuments({ level: 'error' });

Challenge

Easy

A shelf of ten thousand books cannot go out in one response. GET /books now sends one page at a time. It also says how many books match in all, so a client can show "page 1 of 5".

In server.js, change GET /books. Keep the author, from and to filters.
Read ?page= (default 1) and ?limit= (default 10). When either one is missing or not a number, use its default. Page 1 starts at the first book, so page p skips (p - 1) * limit books and then returns the next limit books. Let the database do the skipping and the cutting.
Count total in the database, using the same filter. It counts every matching book, not only the books on this page.
Respond with status 200 and an object, not an array. GET /books?limit=2 returns:
```
{
  "items": [
    { "_id": "6720a1f4c2b9e81d3a5f0c11", "title": "Dune", "author": "Frank Herbert", "year": 1965, "isbn": "9780441013593", "genre": "scifi" },
    { "_id": "6720a1f4c2b9e81d3a5f0c12", "title": "Kindred", "author": "Octavia E. Butler", "year": 1979, "isbn": "9780807083697", "genre": "scifi" }
  ],
  "total": 3
}
```
Try it: Send GET /books?limit=2: you get Dune and Kindred, with "total": 3. Send GET /books?limit=2&page=2: you get only Emma, still with "total": 3. Send GET /books?limit=2&page=3: you get "items": [] and "total": 3. Send GET /books?author=Jane%20Austen&limit=1: you get Emma, with "total": 1. Send GET /books: you get all three books in items.

Try it yourself

import express from 'express';
import { ObjectId } from 'mongodb';
import { books, ratings } from './db.js';

const app = express();
app.use(express.json());

// Every error answers { error: message }.
function sendError(res, status, message) {
    res.status(status).json({ error: message });
}

// The :id as an ObjectId, or null after answering 400 for a bad id.
function parseId(req, res) {
    if (ObjectId.isValid(req.params.id)) {
        return new ObjectId(req.params.id);
    }
    sendError(res, 400, 'Not a valid id');
    return null;
}

app.get('/', (req, res) => {
    res.send('Bookshelf API');
});

app.get('/books', async (req, res) => {
    const filter = {};
    if (req.query.author) {
        filter.author = req.query.author;
    }
    if (req.query.from || req.query.to) {
        filter.year = {};
        if (req.query.from) {
            filter.year.$gte = Number(req.query.from);
        }
        if (req.query.to) {
            filter.year.$lte = Number(req.query.to);
        }
    }
    const list = await books.find(filter).toArray();
    res.json(list);
});

app.post('/books', async (req, res) => {
    const book = req.body;
    try {
        await books.insertOne(book);
    } catch (err) {
        if (err.code === 11000) {
            return sendError(res, 409, 'That ISBN is already on the shelf');
        }
        throw err;
    }
    res.status(201).json(book);
});

app.get('/books/newest', async (req, res) => {
    const n = Number(req.query.n) || 3;
    const list = await books
        .find({}, { projection: { title: 1, year: 1, _id: 0 } })
        .sort({ year: -1 })
        .limit(n)
        .toArray();
    res.json(list);
});

app.get('/books/:id', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    const book = await books.findOne({ _id: id });
    if (!book) {
        return sendError(res, 404, 'No such book');
    }
    res.json(book);
});

app.patch('/books/:id', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    const result = await books.updateOne({ _id: id }, { $set: req.body });
    if (result.matchedCount === 0) {
        return sendError(res, 404, 'No such book');
    }
    res.json(await books.findOne({ _id: id }));
});

app.post('/books/:id/copies', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    const result = await books.updateOne({ _id: id }, { $inc: { copies: 1 } });
    if (result.matchedCount === 0) {
        return sendError(res, 404, 'No such book');
    }
    res.json(await books.findOne({ _id: id }));
});

app.post('/books/:id/tags', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    const result = await books.updateOne({ _id: id }, { $push: { tags: req.body.tag } });
    if (result.matchedCount === 0) {
        return sendError(res, 404, 'No such book');
    }
    res.json(await books.findOne({ _id: id }));
});

app.delete('/books/:id', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    const result = await books.deleteOne({ _id: id });
    if (result.deletedCount === 0) {
        return sendError(res, 404, 'No such book');
    }
    res.status(204).end();
});

app.put('/books/:id/ratings', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    const { user, stars } = req.body;
    await ratings.updateOne({ bookId: id, user }, { $set: { stars } }, { upsert: true });
    res.json({ user, stars });
});

app.get('/books/:id/ratings', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    res.json(await ratings.find({ bookId: id }).toArray());
});

app.get('/stats', async (req, res) => {
    const filter = req.query.genre ? { genre: req.query.genre } : {};
    res.json({
        books: await books.countDocuments(filter),
        ratings: await ratings.countDocuments(),
    });
});

app.get('/stats/genres', async (req, res) => {
    const genres = await books
        .aggregate([{ $group: { _id: '$genre', count: { $sum: 1 } } }])
        .toArray();
    res.json(genres);
});

app.get('/stats/top', async (req, res) => {
    const top = await ratings
        .aggregate([
            { $group: { _id: '$bookId', avg: { $avg: '$stars' }, count: { $sum: 1 } } },
            { $match: { count: { $gte: 2 } } },
            { $sort: { avg: -1 } },
            { $limit: 3 },
        ])
        .toArray();
    res.json(top);
});

app.get('/books/:id/full', async (req, res) => {
    const id = parseId(req, res);
    if (!id) return;
    const [book] = await books
        .aggregate([
            { $match: { _id: id } },
            {
                $lookup: {
                    from: 'authors',
                    localField: 'author',
                    foreignField: 'name',
                    as: 'authorInfo',
                },
            },
        ])
        .toArray();
    if (!book) {
        return sendError(res, 404, 'No such book');
    }
    res.json(book);
});

app.listen(3000, () => {
    console.log('Bookshelf is running on port 3000');
});

All lessons in Databases with MongoDB

Practice on your own: Express playground