MongoDB persistence

by Aleksey Dvoykin

Introduction

In our previous tutorials, we took a look at in-memory and file persistence component implementations. Another frequent choice of persistence is Pip.Service’s MongoDb persistence. This persistence stores data in MongoDB - a popular document-oriented database. The most basic implementation of this component is the MongoDbPersistence class defined in the MongoDb module. It is capable of storing a collection of documents, opening and closing connections, and performing a few simple CRUD operations.

MongoDBPersistence

This is a basic component that stores data items of any type. Some basic operations for creating, getting, and deleting are already included. More advanced CRUD operations over the data items can be implemented in child classes by accessing the collection or model properties. This component also contains methods for opening and closing connections using the credentials provided.

The example below demonstrates a class that implements the MongoDB persistence component for the Beacon data model.

See: MongoDb module


import { FilterParams } from 'pip-services3-commons-nodex';
import { PagingParams } from 'pip-services3-commons-nodex';
import { DataPage } from 'pip-services3-commons-nodex';

import { IdentifiableMongoDbPersistence } from 'pip-services3-mongodb-nodex';

import { BeaconV1 } from '../data/version1/BeaconV1';
import { IBeaconsPersistence } from './IBeaconsPersistence';

export class BeaconsMongoDbPersistence
    extends IdentifiableMongoDbPersistence<BeaconV1, string>
    implements IBeaconsPersistence {

    constructor() {
        super('beacons');
        this._maxPageSize = 1000;
    }

    public async getOneByUdi(correlationId: string, udi: string): Promise<BeaconV1> {
        let criteria = {
            udi: udi
        };

        return new Promise((resolve, reject) => {
            this._collection.findOne(criteria, (err, item) => {
                if (err != null) {
                    reject(err);
                    return;
                }

                if (item != null) this._logger.trace(correlationId, "Found beacon by %s", udi);
                else this._logger.trace(correlationId, "Cannot find beacon by %s", udi);
                
                item = this.convertToPublic(item);
                resolve(item);
            });    
        });     
    }
}

See: MongoDb module

class BeaconMongoDbPersistence : MongoDbPersistence<BeaconV1>
{
    public BeaconMongoDbPersistence()
    {
        base("beacons");
    }

    public async BeaconV1 GetByNameAsync(string correlationId, string name)
    {
        var builder = Builders<BeaconV1>.Filter;
        var filter = builder.Eq(x => x.Name, name);
        var result = await _collection.Find(filter).FirstOrDefaultAsync();
        return result;
    }

    public async BeaconV1 setAsync(String correlatonId, BeaconV1 item)
    {
        var filter = Builders<BeaconV1>.Filter.Eq(x => x.Id, item.Id);
        var options = new FindOneAndReplaceOptions<T>
        {
            ReturnDocument = ReturnDocument.After,
            IsUpsert = true
        };
        var result = await _collection.FindOneAndReplaceAsync(filter, item, options);
        return result;
    }
}

See: MongoDb module

package persistence

import (
	"reflect"
	"strings"

	data1 "github.com/pip-services-samples/service-beacons-gox/data/version1"
	cdata "github.com/pip-services3-gox/pip-services3-commons-gox/data"
	mpersist "github.com/pip-services3-gox/pip-services3-mongodb-gox/persistence"
	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/mongo"
)

type BeaconsMongoDbPersistence struct {
	*mpersist.IdentifiableMongoDbPersistence
}

func NewBeaconsMongoDbPersistence() *BeaconsMongoDbPersistence {
	c := &BeaconsMongoDbPersistence{}
	c.IdentifiableMongoDbPersistence = mpersist.InheritIdentifiableMongoDbPersistence(c, "beacons")
	return c
}

func (c *BeaconsMongoDbPersistence) composeFilter(filter *cdata.FilterParams) interface{} {
	if filter == nil {
		filter = cdata.NewEmptyFilterParams()
	}

	criteria := make([]bson.M, 0, 0)

	id := filter.GetAsString("id")
	if id != "" {
		criteria = append(criteria, bson.M{"_id": id})
	}

	siteId := filter.GetAsString("site_id")
	if siteId != "" {
		criteria = append(criteria, bson.M{"site_id": siteId})
	}
	label := filter.GetAsString("label")
	if label != "" {
		criteria = append(criteria, bson.M{"label": label})
	}
	udi := filter.GetAsString("udi")
	if udi != "" {
		criteria = append(criteria, bson.M{"udi": udi})
	}

	udis := filter.GetAsString("udis")
	var arrUdis []string = make([]string, 0, 0)
	if udis != "" {
		arrUdis = strings.Split(udis, ",")
		if len(arrUdis) > 1 {
			criteria = append(criteria, bson.M{"udi": bson.D{{"$in", arrUdis}}})
		}
	}
	if len(criteria) > 0 {
		return bson.D{{"$and", criteria}}
	}
	return bson.M{}
}

func (c *BeaconsMongoDbPersistence) GetPageByFilter(ctx context.Context, correlationId string, filter *cdata.FilterParams, paging *cdata.PagingParams) (page *cdata.DataPage[BeaconV1], err error) {
	return c.IdentifiableMongoDbPersistence.GetPageByFilter(ctx, correlationId, c.composeFilter(filter), paging, nil, nil)
}

func (c *BeaconsMongoPersistence) GetOneByUdi(ctx context.Context, correlationId string, udi string) (data1.BeaconV1, error) {

	paging := *cdata.NewPagingParams(0, 1, false)
	page, err := c.IdentifiableMongoDbPersistence.GetPageByFilter(ctx, correlationId,
		bson.M{"udi": udi}, paging,
		nil, nil,
	)
	if err != nil {
		return data1.BeaconV1{}, err
	}
	if page.HasData() {
		return page.Data[0], nil
	}
	return data1.BeaconV1{}, nil
}

See: MongoDb module

import 'dart:async';
import 'package:mongo_dart_query/mongo_dart_query.dart' as mngquery;
import 'package:pip_services3_commons/pip_services3_commons.dart';
import 'package:pip_services3_mongodb/pip_services3_mongodb.dart';

import '../data/version1/BeaconV1.dart';
import './IBeaconsPersistence.dart';

class BeaconsMongoDbPersistence
    extends IdentifiableMongoDbPersistence<BeaconV1, String>
    implements IBeaconsPersistence {
  BeaconsMongoDbPersistence() : super('beacons') {
    maxPageSize = 1000;
  }

  dynamic composeFilter(FilterParams? filter) {
    filter = filter ?? FilterParams();

    var criteria = [];

    var id = filter.getAsNullableString('id');
    if (id != null) {
      criteria.add({'_id': id});
    }

    var siteId = filter.getAsNullableString('site_id');
    if (siteId != null) {
      criteria.add({'site_id': siteId});
    }

    var label = filter.getAsNullableString('label');
    if (label != null) {
      criteria.add({'label': label});
    }

    var udi = filter.getAsNullableString('udi');
    if (udi != null) {
      criteria.add({'udi': udi});
    }

    var labelLike = filter.getAsNullableString('label_like');
    if (labelLike != null) {
      var regexp = RegExp(r'^' + labelLike, caseSensitive: false);
      criteria.add({r'$regex': regexp.pattern});
    }

    var udis = filter.getAsObject('udis');
    if (udis is String) {
      udis = udis.split(',');
    }
    if (udis is List) {
      criteria.add({
        'udi': {r'$in': udis}
      });
    }

    return criteria.isNotEmpty ? {r'$and': criteria} : null;
  }

  @override
  Future<DataPage<BeaconV1>> getPageByFilter(
      String? correlationId, FilterParams? filter, PagingParams? paging) async {
    return super
        .getPageByFilterEx(correlationId, composeFilter(filter), paging, null);
  }

  @override
  Future<BeaconV1?> getOneByUdi(String? correlationId, String udi) async {
    var filter = {'udi': udi};
    var query = mngquery.SelectorBuilder();
    var selector = <String, dynamic>{};
    if (filter != null && filter.isNotEmpty) {
      selector[r'$query'] = filter;
    }
    query.raw(selector);

    var item = await collection!.findOne(filter);

    if (item == null) {
      logger.trace(correlationId, 'Nothing found from %s with id = %s',
          [collectionName, udi]);
      return null;
    }
    logger.trace(
        correlationId, 'Retrieved from %s with id = %s', [collectionName, udi]);
    return convertToPublic(item);
  }
}

See: MongoDb module

from typing import Any, Optional

from pip_services3_commons.data import FilterParams, PagingParams, DataPage
from pip_services3_mongodb.persistence import IdentifiableMongoDbPersistence

from .IBeaconsPersistence import IBeaconsPersistence
from ..data.version1 import BeaconV1


class BeaconsMongoDbPersistence(IdentifiableMongoDbPersistence, IBeaconsPersistence):

    def __init__(self):
        super(BeaconsMongoDbPersistence, self).__init__("beacons")
        self._max_page_size = 1000

    def compose_filter(self, filter: FilterParams) -> Any:
        filter = filter if filter is not None else FilterParams()
        criteria = []

        id = filter.get_as_nullable_string("id")
        if id is not None:
            criteria.append({"id": id})
        site_id = filter.get_as_nullable_string("site_id")
        if site_id is not None:
            criteria.append({"site_id": site_id})
        label = filter.get_as_nullable_string("label")
        if label is not None:
            criteria.append({"label": label})
        udi = filter.get_as_nullable_string("udi")
        if udi is not None:
            criteria.append({"udi": udi})
        udis = filter.get_as_object("udis")
        if udis is not None and len(udis) > 0:
            udis = udis.split(",")
            criteria.append({"udi": {"$in": udis}})
        return {"$and": criteria} if len(criteria) > 0 else None

    def get_page_by_filter(self, correlation_id: Optional[str], filter: FilterParams, paging: PagingParams,
                           sort: Any = None, select: Any = None) -> DataPage:
        filter = filter if filter is not None else FilterParams()
        return super(BeaconsMongoDbPersistence, self).get_page_by_filter(correlation_id, self.compose_filter(filter),
                                                                         paging, None, None)

    def get_one_by_udi(self, correlation_id: Optional[str], udi: str) -> BeaconV1:
        if udi is None:
            return None
        item = self._collection.find_one({'udi': udi})
        item = self._convert_to_public(item)

        if item is None:
            self._logger.trace(correlation_id, "Found beacon by %s", udi)
        else:
            self._logger.trace(correlation_id, "Cannot find beacon by %s", udi)

        return item

Not available

And this is how we could use such a class:


let persistence = new BeaconsMongoDbPersistence();
// ...

await persistence.open("test");

let beacon = <BeaconV1>{
    id: '1', 
    site_id: "0001",
    udi: "0002"

};

await persistence.set("test", beacon);
let item = persistence.getOneByUdi("test", "0002");
await persistence.close("test");
console.log(item.udi); // Result: 0002

var persistence = new BeaconMongoDbPersistence();
// ...

await persistence.OpenAsync("test");

var beacon = new BeaconV1
{
    name: "Super Beacon"
};
await persistence.SetAsync("test", beacon);
var item = await persistence.GetByNameAsync("test", "Super Beacon");
await persistence.CloseAsync("test");
Console.Out.WriteLine(item);                   // Result: { name: "Super Beacon" }

persistence := persist.NewBeaconsMongoDbPersistence()
// ...

persistence.Open("test")

beacon := &data1.BeaconV1{
	Id:  "1",
	Udi: "0002",
	SiteId: "0001",
}

persistence.Set(context.Background(), "test", beacon)
item, _ := persistence.GetOneByUdi(context.Background(), "test", "0002")
persistence.Close(context.Background(), "test")
fmt.Println(item.Id)

var persistence = BeaconsMongoDbPersistence();
//...

await persistence.open('test');

var beacon = BeaconV1(id: '1', site_id: '0001', udi: '0002');

await persistence.set('test', beacon);
var item = await persistence.getOneByUdi('test', '0002');
await persistence.close('test');
print(item!.udi); // Result: 0002

persistence = BeaconsMongoDbPersistence()
# ...

persistence.open("test")

beacon = BeaconV1(id="1", site_id="0001", udi="0002")

persistence.set("test", beacon)
item = persistence.get_one_by_udi("test", "1")
persistence.close("test")
print(item)

Not available

Configuring database connections

As mentioned earlier, the MongoDbPersistence contains methods for opening and closing connections. To connect to the appropriate database and collection, we need to first configure the connection with all necessary parameters. MongoDbPersistence uses the MongoDbConnection class for establishing connections.

The MongoDbConnection class provides MongoDB connectivity using a plain driver. To reduce the number of database connections needed, a connection can be defined and then shared through multiple persistence components.

By default, MongoDbPersistence tries to establish a local connection on MongoDb’s default port - 27017. If the desired MongoDb server is located elsewhere, the persistence should be configured with the corresponding host and port information. Persistence configuration can be performed in a number of ways.

The example below demonstrates how the ConfigParams class can be used for persistence configuration. To learn more about this class, and about microservice configuration in general, be sure to read this.

See: MongoDbPersistence, MongoDbConnection, ConfigParams


let persistence = new BeaconsMongoDbPersistence();
// Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000

persistence.configure(ConfigParams.fromTuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test"
))
await persistence.open(null) // While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000

See: MongoDbPersistence, MongoDbConnection, ConfigParams

var persistence = new BeaconMongoDbPersistence();
// Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000

persistence.Configure(ConfigParams.FromTuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test"
));
await persistence.openAsync(); 	// While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000

See: MongoDbPersistence, MongoDbConnection, ConfigParams

persistence := persist.NewBeaconsMongoDbPersistence()

// Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000
persistence.Configure(context.Background(), cconf.NewConfigParamsFromTuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test",
))

// While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000
persistence.Open(context.Background(), "123")

See: MongoDbPersistence, MongoDbConnection, ConfigParams

var persistence = BeaconsMongoDbPersistence();
// Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000

persistence.configure(ConfigParams.fromTuples([
  'connection.host',
  'localhost',
  'connection.port',
  '30000',
  'connection.database',
  'test'
]));

// While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000
await persistence.open(null);

See: MongoDbPersistence, MongoDbConnection, ConfigParams

persistence = BeaconsMongoDbPersistence()
# Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000

persistence.configure(ConfigParams.from_tuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test"
))
persistence.open(None) # While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000

Not available

Likewise, a connection can be configured using a configuration file. In this case, there exist two approaches:

configuring multiple persistences using a common MongoDbConnection.
configuring a single persistence with its own, private MongoDbConnection.

To perform configuration using a single MongoDbConnection, one of the following descriptors should be used:

pip-services:connection:mongodb:*:1.0 or pip-services3:connection:mongodb:*:1.0.

To learn more about references, descriptors, and component references, follow this link.
First, add an element with the “pip-services” descriptor to the configuration file.

...
# MongoDb Connection
- descriptor: "pip-services:connection:mongodb:default:1.0"
  connection:
    host: localhost
    port: 30000
...

Next, register the persistence as a component in the microservice’s Factory:


export class BeaconsServiceFactory extends Factory{
    public static MemoryPersistenceDescriptor = new Descriptor('beacons', 'persistence', 'memory', '*', '1.0');
    public static FilePersistenceDescriptor = new Descriptor('beacons', 'persistence', 'file', '*', '1.0');
    public static MongoDbPersistenceDescriptor = new Descriptor('beacons', 'persistence', 'mongodb', '*', '1.0');
    public static ControllerDescriptor = new Descriptor('beacons', 'controller', 'default', '*', '1.0');
    public static HttpServiceV1Descriptor = new Descriptor('beacons', 'service', 'http', '*', '1.0');
    
    constructor(){
        super();

        this.registerAsType(BeaconsServiceFactory.MemoryPersistenceDescriptor, BeaconsMemoryPersistence);
        this.registerAsType(BeaconsServiceFactory.FilePersistenceDescriptor, BeaconsFilePersistence);
        this.registerAsType(BeaconsServiceFactory.MongoDbPersistenceDescriptor, BeaconsMongoDbPersistence);
        this.registerAsType(BeaconsServiceFactory.ControllerDescriptor, BeaconsController);
        this.registerAsType(BeaconsServiceFactory.HttpServiceV1Descriptor, BeaconsHttpServiceV1);
    }
}

public class BeaconsServiceFactory : Factory
{
    public static Descriptor Descriptor = new Descriptor("beacons", "factory", "service", "default", "1.0");
    public static Descriptor MemoryPersistenceDescriptor = new Descriptor("beacons", "persistence", "memory", "*", "1.0");
    public static Descriptor MongoDbPersistenceDescriptor = new Descriptor("beacons", "persistence", "mongodb", "*", "1.0");
    public static Descriptor ControllerDescriptor = new Descriptor("beacons", "controller", "default", "*", "1.0");
    public static Descriptor HttpServiceDescriptor = new Descriptor("beacons", "service", "http", "*", "1.0");
    public BeaconsServiceFactory()
    {
        RegisterAsType(MemoryPersistenceDescriptor, typeof(BeaconsMemoryPersistence));
        RegisterAsType(MongoDbPersistenceDescriptor, typeof(BeaconsMongoDbPersistence));
        RegisterAsType(ControllerDescriptor, typeof(BeaconsController));
        RegisterAsType(HttpServiceDescriptor, typeof(BeaconsHttpServiceV1));
    }
}

import (
	logic "github.com/pip-services-samples/service-beacons-gox/logic"
	persist "github.com/pip-services-samples/service-beacons-gox/persistence"
	services1 "github.com/pip-services-samples/service-beacons-gox/services/version1"
	cref "github.com/pip-services3-gox/pip-services3-commons-gox/refer"
	cbuild "github.com/pip-services3-gox/pip-services3-components-gox/build"
)

type BeaconsServiceFactory struct {
	*cbuild.Factory
}

func NewBeaconsServiceFactory() *BeaconsServiceFactory {
	c := &BeaconsServiceFactory{
		Factory: cbuild.NewFactory(),
	}

	memoryPersistenceDescriptor := cref.NewDescriptor("beacons", "persistence", "memory", "*", "1.0")
	filePersistenceDescriptor := cref.NewDescriptor("beacons", "persistence", "file", "*", "1.0")
	mongoDbPersistenceDescriptor := cref.NewDescriptor("beacons", "persistence", "mongodb", "*", "1.0")
	controllerDescriptor := cref.NewDescriptor("beacons", "controller", "default", "*", "1.0")
	httpServiceV1Descriptor := cref.NewDescriptor("beacons", "service", "http", "*", "1.0")

	c.RegisterType(memoryPersistenceDescriptor, persist.NewBeaconsMemoryPersistence)
	c.RegisterType(filePersistenceDescriptor, persist.NewBeaconsFilePersistence)
	c.RegisterType(mongoDbPersistenceDescriptor, persist.NewBeaconsMongoDbPersistence)
	c.RegisterType(controllerDescriptor, logic.NewBeaconsController)
	c.RegisterType(httpServiceV1Descriptor, services1.NewBeaconsHttpServiceV1)

	return c
}

import 'package:pip_services3_components/pip_services3_components.dart';
import 'package:pip_services3_commons/pip_services3_commons.dart';

import '../persistence/BeaconsMemoryPersistence.dart';
import '../persistence/BeaconsFilePersistence.dart';
import '../persistence/BeaconsMongoDbPersistence.dart';
import '../logic/BeaconsController.dart';
import '../services/version1/BeaconsCommandableHttpServiceV1.dart';

class BeaconsServiceFactory extends Factory {
  static final MemoryPersistenceDescriptor =
      Descriptor('beacons', 'persistence', 'memory', '*', '1.0');
  static final FilePersistenceDescriptor =
      Descriptor('beacons', 'persistence', 'file', '*', '1.0');
  static final MongoDbPersistenceDescriptor =
      Descriptor('beacons', 'persistence', 'mongodb', '*', '1.0');
  static final ControllerDescriptor =
      Descriptor('beacons', 'controller', 'default', '*', '1.0');
  static final CommandableHttpServiceV1Descriptor =
      Descriptor('beacons', 'service', 'commandable-http', '*', '1.0');
  
  BeaconsServiceFactory() : super() {
    registerAsType(BeaconsServiceFactory.MemoryPersistenceDescriptor,
        BeaconsMemoryPersistence);
    registerAsType(BeaconsServiceFactory.FilePersistenceDescriptor,
        BeaconsFilePersistence);
    registerAsType(BeaconsServiceFactory.MongoDbPersistenceDescriptor,
        BeaconsMongoDbPersistence);
    registerAsType(
        BeaconsServiceFactory.ControllerDescriptor, BeaconsController);
    registerAsType(BeaconsServiceFactory.CommandableHttpServiceV1Descriptor,
        BeaconsCommandableHttpServiceV1);
    
  }
}

 class BeaconsServiceFactory(Factory):

    MemoryPersistenceDescriptor = Descriptor('beacons', 'persistence', 'memory', '*', '1.0')
    FilePersistenceDescriptor = Descriptor('beacons', 'persistence', 'file', '*', '1.0')
    MongoDbPersistenceDescriptor = Descriptor('beacons', 'persistence', 'mongodb', '*', '1.0')
    ControllerDescriptor = Descriptor('beacons', 'controller', 'default', '*', '1.0')
    HttpServiceV1Descriptor = Descriptor('beacons', 'service', 'http', '*', '1.0')

    def __init__(self):
        super(BeaconsServiceFactory, self).__init__()

        self.register_as_type(BeaconsServiceFactory.MemoryPersistenceDescriptor, BeaconsMemoryPersistence)
        self.register_as_type(BeaconsServiceFactory.FilePersistenceDescriptor, BeaconsFilePersistence)
        self.register_as_type(BeaconsServiceFactory.MongoDbPersistenceDescriptor, BeaconsMongoDbPersistence)
        self.register_as_type(BeaconsServiceFactory.ControllerDescriptor, BeaconsController)
        self.register_as_type(BeaconsServiceFactory.HttpServiceV1Descriptor, BeaconsHttpServiceV1)

Not available