MongoDB persistence

by Aleksey Dvoykin

Introduction

In our previous tutorials, we took a look at in-memory and file persistence component implementations. Another frequent choice of persistence is Pip.Service’s MongoDb persistence. This persistence stores data in MongoDB - a popular document-oriented database. The most basic implementation of this component is the MongoDbPersistence class defined in the MongoDb module. It is capable of storing a collection of documents, opening and closing connections, and performing a few simple CRUD operations.

MongoDBPersistence

This is a basic component that stores data items of any type. Some basic operations for creating, getting, and deleting are already included. More advanced CRUD operations over the data items can be implemented in child classes by accessing the collection or model properties. This component also contains methods for opening and closing connections using the credentials provided.

The example below demonstrates a class that implements the MongoDB persistence component for the Beacon data model.


import { FilterParams } from 'pip-services4-data-node';
import { PagingParams } from 'pip-services4-data-node';
import { DataPage } from 'pip-services4-data-node';
import { Context } from 'pip-services4-components-node';

import { IdentifiableMongoDbPersistence } from 'pip-services4-mongodb-node';

import { BeaconV1 } from '../data/version1/BeaconV1';
import { IBeaconsPersistence } from './IBeaconsPersistence';

export class BeaconsMongoDbPersistence
    extends IdentifiableMongoDbPersistence<BeaconV1, string>
    implements IBeaconsPersistence {

    constructor() {
        super('beacons');
        this._maxPageSize = 1000;
    }

    public async getOneByUdi(ctx: Context, udi: string): Promise<BeaconV1> {
        let criteria = {
            udi: udi
        };

        return new Promise((resolve, reject) => {
            this._collection.findOne(criteria, (err, item) => {
                if (err != null) {
                    reject(err);
                    return;
                }

                if (item != null) this._logger.trace(ctx, "Found beacon by %s", udi);
                else this._logger.trace(ctx, "Cannot find beacon by %s", udi);
                
                item = this.convertToPublic(item);
                resolve(item);
            });    
        });     
    }
}

See: MongoDb module

package persistence

import (
	"context"
	"strings"

	data1 "github.com/pip-services-samples/service-beacons-go/data/version1"
	cquery "github.com/pip-services4/pip-services4-go/pip-services4-data-go/query"
	cmongo "github.com/pip-services4/pip-services4-go/pip-services4-mongodb-go/persistence"
	"go.mongodb.org/mongo-driver/bson"
)

type BeaconsMongoPersistence struct {
	cmongo.IdentifiableMongoDbPersistence[data1.BeaconV1, string]
}

func NewBeaconsMongoPersistence() *BeaconsMongoPersistence {
	c := &BeaconsMongoPersistence{}
	c.IdentifiableMongoDbPersistence = *cmongo.InheritIdentifiableMongoDbPersistence[data1.BeaconV1, string](c, "beacons")
	return c
}

func (c *BeaconsMongoPersistence) GetPageByFilter(ctx context.Context, filter cquery.FilterParams, paging cquery.PagingParams) (cquery.DataPage[data1.BeaconV1], error) {
	filterObj := bson.M{}
	if id, ok := filter.GetAsNullableString("id"); ok && id != "" {
		filterObj["_id"] = id
	}
	if siteId, ok := filter.GetAsNullableString("site_id"); ok && siteId != "" {
		filterObj["site_id"] = siteId
	}
	if typeId, ok := filter.GetAsNullableString("type"); ok && typeId != "" {
		filterObj["type"] = typeId
	}
	if udi, ok := filter.GetAsNullableString("udi"); ok && udi != "" {
		filterObj["udi"] = udi
	}
	if label, ok := filter.GetAsNullableString("label"); ok && label != "" {
		filterObj["label"] = label
	}
	if udis, ok := filter.GetAsObject("udis"); ok {
		var udisM bson.M
		switch _udis := udis.(type) {
		case []string:
			if len(_udis) > 0 {
				udisM = bson.M{"$in": _udis}
			}
			break
		case string:
			if _udisArr := strings.Split(_udis, ","); len(_udisArr) > 0 {
				udisM = bson.M{"$in": _udisArr}
			}
			break
		}
		if udisM != nil {
			filterObj["udi"] = udisM
		}
	}

	return c.IdentifiableMongoDbPersistence.GetPageByFilter(ctx,
		filterObj, paging,
		nil, nil,
	)
}

func (c *BeaconsMongoPersistence) GetOneByUdi(ctx context.Context, udi string) (data1.BeaconV1, error) {

	paging := *cquery.NewPagingParams(0, 1, false)
	page, err := c.IdentifiableMongoDbPersistence.GetPageByFilter(ctx,
		bson.M{"udi": udi}, paging,
		nil, nil,
	)
	if err != nil {
		return data1.BeaconV1{}, err
	}
	if page.HasData() {
		return page.Data[0], nil
	}
	return data1.BeaconV1{}, nil
}

See: MongoDb module


from .IBeaconsPersistence import IBeaconsPersistence
from ..data.version1 import BeaconV1


class BeaconsMongoDbPersistence(IdentifiableMongoDbPersistence, IBeaconsPersistence):

    def __init__(self):
        super(BeaconsMongoDbPersistence, self).__init__("beacons")
        self._max_page_size = 1000

    def compose_filter(self, filter: FilterParams) -> Any:
        filter = filter if filter is not None else FilterParams()
        criteria = []

        id = filter.get_as_nullable_string("id")
        if id is not None:
            criteria.append({"id": id})
        site_id = filter.get_as_nullable_string("site_id")
        if site_id is not None:
            criteria.append({"site_id": site_id})
        label = filter.get_as_nullable_string("label")
        if label is not None:
            criteria.append({"label": label})
        udi = filter.get_as_nullable_string("udi")
        if udi is not None:
            criteria.append({"udi": udi})
        udis = filter.get_as_object("udis")
        if udis is not None and len(udis) > 0:
            udis = udis.split(",")
            criteria.append({"udi": {"$in": udis}})
        return {"$and": criteria} if len(criteria) > 0 else None

    def get_page_by_filter(self, context: Optional[IContext], filter: FilterParams, paging: PagingParams,
                           sort: Any = None, select: Any = None) -> DataPage:
        filter = filter if filter is not None else FilterParams()
        return super(BeaconsMongoDbPersistence, self).get_page_by_filter(context, self.compose_filter(filter),
                                                                         paging, None, None)

    def get_one_by_udi(self, context: Optional[str], udi: str) -> BeaconV1:
        if udi is None:
            return None
        item = self._collection.find_one({'udi': udi})
        item = self._convert_to_public(item)

        if item is None:
            self._logger.trace(context, "Found beacon by %s", udi)
        else:
            self._logger.trace(context, "Cannot find beacon by %s", udi)

        return item
Not available

And this is how we could use such a class:


let persistence = new BeaconsMongoDbPersistence();
// ...

await persistence.open(ctx);

let beacon = <BeaconV1>{
    id: '1', 
    site_id: "0001",
    udi: "0002"

};

await persistence.set(ctx, beacon);
let item = persistence.getOneByUdi(ctx, "0002");
await persistence.close(ctx);
console.log(item.udi); // Result: 0002






persistence := NewBeaconsMongoPersistence()
// ...

persistence.Open(context.Background())

beacon := data1.BeaconV1{
	Id:     "1",
	Udi:    "0002",
	SiteId: "0001",
}

persistence.Set(context.Background(), beacon)
item, _ := persistence.GetOneByUdi(context.Background(), "0002")
persistence.Close(context.Background())
fmt.Println(item.Id)



persistence = BeaconsMongoDbPersistence()
# ...

persistence.open("test")

beacon = BeaconV1(id="1", site_id="0001", udi="0002")

persistence.set("test", beacon)
item = persistence.get_one_by_udi("test", "1")
persistence.close("test")
print(item)
Not available

Configuring database connections

As mentioned earlier, the MongoDbPersistence contains methods for opening and closing connections. To connect to the appropriate database and collection, we need to first configure the connection with all necessary parameters. MongoDbPersistence uses the MongoDbConnection class for establishing connections.

The MongoDbConnection class provides MongoDB connectivity using a plain driver. To reduce the number of database connections needed, a connection can be defined and then shared through multiple persistence components.

By default, MongoDbPersistence tries to establish a local connection on MongoDb’s default port - 27017. If the desired MongoDb server is located elsewhere, the persistence should be configured with the corresponding host and port information. Persistence configuration can be performed in a number of ways.

The example below demonstrates how the ConfigParams class can be used for persistence configuration. To learn more about this class, and about microservice configuration in general, be sure to read this.


let persistence = new BeaconsMongoDbPersistence();
// Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000

persistence.configure(ConfigParams.fromTuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test"
))
await persistence.open(null) // While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000

See: MongoDbPersistence, MongoDbConnection, ConfigParams

persistence := persist.NewBeaconsMongoDbPersistence()

// Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000
persistence.Configure(context.Background(), cconf.NewConfigParamsFromTuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test",
))

// While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000
persistence.Open(context.Background(), None)

See: MongoDbPersistence, MongoDbConnection, ConfigParams

persistence = BeaconsMongoDbPersistence()
# Let's say we need to connect to a local MongoDb, but on a non-standard port - 30000

persistence.configure(ConfigParams.from_tuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test"
))
persistence.open(None) # While opening, it will try to establish a connection with the locally hosted MongoDb on port 30000
Not available

Likewise, a connection can be configured using a configuration file. In this case, there exist two approaches:

  1. configuring multiple persistences using a common MongoDbConnection.
  2. configuring a single persistence with its own, private MongoDbConnection.

To perform configuration using a single MongoDbConnection, one of the following descriptors should be used:

pip-services:connection:mongodb:*:1.0 or pip-services3:connection:mongodb:*:1.0.

To learn more about references, descriptors, and component references, follow this link.
First, add an element with the “pip-services” descriptor to the configuration file.

...
# MongoDb Connection
- descriptor: "pip-services:connection:mongodb:default:1.0"
  connection:
    host: localhost
    port: 30000
...

Next, register the persistence as a component in the microservice’s Factory:


export class BeaconsServiceFactory extends Factory{
    public static MemoryPersistenceDescriptor = new Descriptor('beacons', 'persistence', 'memory', '*', '1.0');
    public static FilePersistenceDescriptor = new Descriptor('beacons', 'persistence', 'file', '*', '1.0');
    public static MongoDbPersistenceDescriptor = new Descriptor('beacons', 'persistence', 'mongodb', '*', '1.0');
    public static ServiceDescriptor = new Descriptor('beacons', 'service', 'default', '*', '1.0');
    public static HttpControllerV1Descriptor = new Descriptor('beacons', 'controller', 'http', '*', '1.0');
    
    constructor(){
        super();

        this.registerAsType(BeaconsServiceFactory.MemoryPersistenceDescriptor, BeaconsMemoryPersistence);
        this.registerAsType(BeaconsServiceFactory.FilePersistenceDescriptor, BeaconsFilePersistence);
        this.registerAsType(BeaconsServiceFactory.MongoDbPersistenceDescriptor, BeaconsMongoDbPersistence);
        this.registerAsType(BeaconsServiceFactory.ServiceDescriptor, BeaconsService);
        this.registerAsType(BeaconsServiceFactory.HttpControllerV1Descriptor, BeaconsHttpControllerV1);
    }
}








import (
	controllers1 "github.com/pip-services-samples/service-beacons-go/controllers/version1"
	persist "github.com/pip-services-samples/service-beacons-go/persistence"
	logic "github.com/pip-services-samples/service-beacons-go/service"
	cbuild "github.com/pip-services4/pip-services4-go/pip-services4-components-go/build"
	cref "github.com/pip-services4/pip-services4-go/pip-services4-components-go/refer"
)

type BeaconsServiceFactory struct {
	cbuild.Factory
}

func NewBeaconsServiceFactory() *BeaconsServiceFactory {
	c := &BeaconsServiceFactory{
		Factory: *cbuild.NewFactory(),
	}

	memoryPersistenceDescriptor := cref.NewDescriptor("beacons", "persistence", "memory", "*", "1.0")
	mongoPersistenceDescriptor := cref.NewDescriptor("beacons", "persistence", "mongodb", "*", "1.0")
	filePersistenceDescriptor := cref.NewDescriptor("beacons", "persistence", "file", "*", "1.0")
	serviceDescriptor := cref.NewDescriptor("beacons", "service", "default", "*", "1.0")
	httpcontrollerV1Descriptor := cref.NewDescriptor("beacons", "controller", "http", "*", "1.0")

	c.RegisterType(mongoPersistenceDescriptor, persist.NewBeaconsMongoPersistence)
	c.RegisterType(memoryPersistenceDescriptor, persist.NewBeaconsMemoryPersistence)
	c.RegisterType(filePersistenceDescriptor, persist.NewBeaconsFilePersistence)
	c.RegisterType(serviceDescriptor, logic.NewBeaconsService)
	c.RegisterType(httpcontrollerV1Descriptor, controllers1.NewBeaconsHttpControllerV1)

	return c
}



 class BeaconsControllerFactory(Factory):

    MemoryPersistenceDescriptor = Descriptor('beacons', 'persistence', 'memory', '*', '1.0')
    FilePersistenceDescriptor = Descriptor('beacons', 'persistence', 'file', '*', '1.0')
    MongoDbPersistenceDescriptor = Descriptor('beacons', 'persistence', 'mongodb', '*', '1.0')
    ServiceDescriptor = Descriptor('beacons', 'service', 'default', '*', '1.0')
    HttpControllerV1Descriptor = Descriptor('beacons', 'controller', 'http', '*', '1.0')

    def __init__(self):
        super(BeaconsControllerFactory, self).__init__()

        self.register_as_type(BeaconsControllerFactory.MemoryPersistenceDescriptor, BeaconsMemoryPersistence)
        self.register_as_type(BeaconsControllerFactory.FilePersistenceDescriptor, BeaconsFilePersistence)
        self.register_as_type(BeaconsControllerFactory.MongoDbPersistenceDescriptor, BeaconsMongoDbPersistence)
        self.register_as_type(BeaconsControllerFactory.ControllerDescriptor, BeaconsService)
        self.register_as_type(BeaconsControllerFactory.HttpControllerV1Descriptor, BeaconsHttpControllerV1)
Not available 

export class BeaconsProcess extends ProcessContainer{
    public constructor(){
        super('beacons', 'Beacons microservice');

        this.addFactory(new BeaconsServiceFactory());
        this.addFactory(new DefaultHttpFactory());
        this.addFactory(new DefaultSwaggerFactory());
    }
}

And add the DefaultMongoDbFactory to the microservice’s ProcessContainer:

import (
	factory "github.com/pip-services-samples/service-beacons-go/build"
	cproc "github.com/pip-services4/pip-services4-go/pip-services4-container-go/container"
	rbuild "github.com/pip-services4/pip-services4-go/pip-services4-http-go/build"
	cpg "github.com/pip-services4/pip-services4-go/pip-services4-postgres-go/build"
)

type BeaconsProcess struct {
	cproc.ProcessContainer
}

func NewBeaconsProcess() *BeaconsProcess {
	c := &BeaconsProcess{
		ProcessContainer: *cproc.NewProcessContainer("beacons", "Beacons microservice"),
	}

	c.AddFactory(factory.NewBeaconsServiceFactory())
	c.AddFactory(rbuild.NewDefaultHttpFactory())
	c.AddFactory(cpg.NewDefaultPostgresFactory())

	return c
}

And add the DefaultMongoDbFactory to the microservice’s ProcessContainer:

 class BeaconsProcess(ProcessContainer):
    def __init__(self):
        super(BeaconsProcess, self).__init__('beacons', 'Beacons microservice')

        self._factories.add(BeaconsControllerFactory())
        self._factories.add(DefaultRpcFactory())
        self._factories.add(DefaultSwaggerFactory())
Not available

If we’re configuring just a single connection to the Beacons MongoDB persistence, the connection configuration should use the “beacons” descriptor:

...
# MongoDb persistence
- descriptor: "beacons:persistence:mongodb:default:1.0"
  connection:
    host: localhost
    port: 30000
...

Identifiable data objects and IdentifiableMongoDBPersistence


export interface IIdentifiable<K> {
	/** The unique object identifier of type K. */
	id: K;
}

The implementation we will be working with going forward is called the IdentifiableMongoDbPersistence. It stores and processes data objects that have a unique ID field and implement the IIdentifiable interface defined in the Commons module.

type IIdentifiable[K any] interface {
	GetId() K
}

The implementation we will be working with going forward is called the IdentifiableMongoDbPersistence. It stores and processes data objects that have a unique ID field and implement the IIdentifiable interface defined in the Commons module.

class IIdentifiable(ABC):
    id: Any
Not available

IdentifiableMongoDbPersistence implements a number of CRUD operations that are based on working with the model’s id in a predefined manner. In addition, it provides methods for getting paginated results and listing data using detailed filter, sort, and even projection parameters.


export class IdentifiableMongoDbPersistence<T extends IIdentifiable<K>, K> extends MongoDbPersistence<T>
    implements IWriter<T, K>, IGetter<T, K>, ISetter<T> {

    public constructor(collection: string);

    protected async convertFromPublicPartial(value: any): any;

    public async getListByIds(ctx: Context, ids: K[]): Promise<T[]>;

    public async getOneByUdi(ctx: Context, id: K): Promise<T>;

    public async create(ctx: Context, item: T): Promise<T>;

    public async set(ctx: Context, item: T): Promise<T>;

    public async update(ctx: Context, item: T): Promise<T>;

    public async updatePartially(ctx: Context, id: K, data: AnyValueMap): Promise<T> ;

    public async deleteById(ctx: Context, id: K): Promise<T> ;

    public async deleteByIds(ctx: Context, ids: K[]): Promise<void>;
}






type IdentifiableMongoDbPersistence struct {
	*MongoDbPersistence
}

func InheritIdentifiableMongoDbPersistence(overrides IMongoDbPersistenceOverrides) *IdentifiableMongoDbPersistence[T, K] {
    ...
}

func (c *IdentifiableMongoDbPersistence) Configure(ctx context.Context, config *cconf.ConfigParams) {
    ...
}

func (c *IdentifiableMongoDbPersistence) GetListByIds(ctx context.Context, ids []K) (items []T, err error) {
    ...
}

func (c *IdentifiableMongoDbPersistence) GetOneById(ctx context.Context, id K) (item T, err error) {
    ...
}

func (c *IdentifiableMongoDbPersistence) Create(ctx context.Context, item T) (result T, err error) {
    ...
}

func (c *IdentifiableMongoDbPersistence) Set(ctx context.Context, item T) (result T, err error) {
    ...
}

func (c *IdentifiableMongoDbPersistence) Update(ctx context.Context, item T) (result T, err error) {
    ...
}

func (c *IdentifiableMongoDbPersistence) UpdatePartially(ctx context.Context, id K, data *cdata.AnyValueMap) (item T, err error) {
    ...
}

func (c *IdentifiableMongoDbPersistence) DeleteById(ctx context.Context, id K) (item T, err error) {
    ...
}

func (c *IdentifiableMongoDbPersistence) DeleteByIds(ctx context.Context, ids []K) error {
    ...
}



class IdentifiableMongoDbPersistence(MongoDbPersistence):

    def __init__(self, collection: str = None):
        ...

    def _convert_from_public_partial(self, value: Any) -> Any:
        ...

    def get_list_by_ids(self, context: Optional[IContext], ids: List[Any]) -> List[T]:
        ...

    def get_one_by_udi(self, context: Optional[IContext], id: Any) -> T:
        ...

    def create(self, context: Optional[IContext], item: T) -> T:
        ...

    def set(self, context: Optional[IContext], item: T) -> T:
        ...

    def update(self, context: Optional[IContext], item: T) -> Optional[T]:
        ...

    def update_partially(self, context: Optional[IContext], id: Any, data: AnyValueMap) -> T:
        ...

    def delete_by_id(self, context: Optional[IContext], id: Any) -> T:
        ...

    def delete_by_ids(self, context: Optional[IContext], ids: List[Any]):
        ...
Not available

We can build upon the IdentifiableMongoDbPersistence by overriding its ComposeFilter method:


export class BeaconsMongoDbPersistence
    extends IdentifiableMongoDbPersistence<BeaconV1, string>
    implements IBeaconsPersistence {

    constructor() {
        super('beacons');
        this._maxPageSize = 1000;
    }

    private composeFilter(filter: FilterParams): any {
        filter = filter || new FilterParams();

        let criteria = [];

        let id = filter.getAsNullableString('id');
        if (id != null) {
            criteria.push({ _id: id });
        }

        let siteId = filter.getAsNullableString('site_id');
        if (siteId != null) {
            criteria.push({ site_id: siteId });
        }

        let label = filter.getAsNullableString('label');
        if (label != null) {
            criteria.push({ label: label });
        }

        let udi = filter.getAsNullableString('udi');
        if (udi != null) {
            criteria.push({ udi: udi });
        }

        let udis = filter.getAsObject('udis');
        if (typeof udis === "string") {
            udis = udis.split(',');
        }
        if (Array.isArray(udis)) {
            criteria.push({ udi: { $in: udis } });
        }

        return criteria.length > 0 ? { $and: criteria } : null;
    }

    public async getPageByFilter(ctx: Context, filter: FilterParams,
        paging: PagingParams): Promise<DataPage<BeaconV1>> {
        return await super.getPageByFilter(ctx, this.composeFilter(filter), paging, null, null);
    }







import (
	"context"
	"strings"

	data1 "github.com/pip-services-samples/service-beacons-go/data/version1"
	cquery "github.com/pip-services4/pip-services4-go/pip-services4-data-go/query"
	cmongo "github.com/pip-services4/pip-services4-go/pip-services4-mongodb-go/persistence"
	"go.mongodb.org/mongo-driver/bson"
)

type BeaconsMongoDbPersistence struct {
	cmongo.IdentifiableMongoDbPersistence[data1.BeaconV1, string]
}

func NewBeaconsMongoDbPersistence() *BeaconsMongoDbPersistence {
	c := &BeaconsMongoDbPersistence{}
	c.IdentifiableMongoDbPersistence = *cmongo.InheritIdentifiableMongoDbPersistence[data1.BeaconV1, string](c, "beacons")
	return c
}

func (c *BeaconsMongoDbPersistence) composeFilter(filter *cquery.FilterParams) interface{} {
	if filter == nil {
		filter = cquery.NewEmptyFilterParams()
	}

	criteria := make([]bson.M, 0, 0)

	id := filter.GetAsString("id")
	if id != "" {
		criteria = append(criteria, bson.M{"_id": id})
	}

	siteId := filter.GetAsString("site_id")
	if siteId != "" {
		criteria = append(criteria, bson.M{"site_id": siteId})
	}
	label := filter.GetAsString("label")
	if label != "" {
		criteria = append(criteria, bson.M{"label": label})
	}
	udi := filter.GetAsString("udi")
	if udi != "" {
		criteria = append(criteria, bson.M{"udi": udi})
	}

	udis := filter.GetAsString("udis")
	var arrUdis []string = make([]string, 0, 0)
	if udis != "" {
		arrUdis = strings.Split(udis, ",")
		if len(arrUdis) > 1 {
			criteria = append(criteria, bson.M{"udi": bson.D{{"$in", arrUdis}}})
		}
	}
	if len(criteria) > 0 {
		return bson.D{{"$and", criteria}}
	}
	return bson.M{}
}

func (c *BeaconsMongoDbPersistence) GetPageByFilter(ctx context.Context, correlationId string, filter *cquery.FilterParams, paging cquery.PagingParams) (page cquery.DataPage[data1.BeaconV1], err error) {
	return c.IdentifiableMongoDbPersistence.GetPageByFilter(ctx, c.composeFilter(filter), paging, nil, nil)
}




 class BeaconsMongoDbPersistence(IdentifiableMongoDbPersistence, IBeaconsPersistence):

    def __init__(self):
        super(BeaconsMongoDbPersistence, self).__init__("beacons")
        self._max_page_size = 1000

    def compose_filter(self, filter: FilterParams) -> Any:
        filter = filter if filter is not None else FilterParams()
        criteria = []

        id = filter.get_as_nullable_string("id")
        if id is not None:
            criteria.append({"id": id})
        site_id = filter.get_as_nullable_string("site_id")
        if site_id is not None:
            criteria.append({"site_id": site_id})
        label = filter.get_as_nullable_string("label")
        if label is not None:
            criteria.append({"label": label})
        udi = filter.get_as_nullable_string("udi")
        if udi is not None:
            criteria.append({"udi": udi})
        udis = filter.get_as_object("udis")
        if udis is not None and len(udis) > 0:
            udis = udis.split(",")
            criteria.append({"udi": {"$in": udis}})
        return {"$and": criteria} if len(criteria) > 0 else None

    def get(self, context: IContext, filter: FilterParams, paging: PagingParams) -> T
        return get_page_by_filter(context, self.__compose_filter(filter), paging, None, None)
Not available

In most scenarios, child classes only need to override the getPageeByFilter(), getListByFilter(), or deleteByFilter() operations using a custom filter function (like the compose_filter function in the example above). All of the other operations can be used straight out of the box. Developers can implement custom methods by directly accessing the data objects, which are stored in the _collection property. See the MongoDb module API documentation for more details.

Filtering

Persistence components in the Pip.Services Toolkit use a number of data patterns. IdentifiableMongoDbPersistence, for example, supports Filtering. This pattern allows clients to use a FilterParams object to describe a subset of data using key-value pairs. These FilterParams can then be used for retrieving data in accordance with the specified search criteria (see the Commons module).

let filter = FilterParams.fromTuples(
    "name", 'ABC'
 )
let page = await persistence.getPageByFilter(ctx, filter, null)

See: MongoDb module, Commons module, FilterParams

filter := cquery.NewFilterParamsFromTuples(
	"name", "ABC",
)
result, _ := persistence.GetPageByFilter(context.Background(), filter, nil)

See: MongoDb module, Commons module, FilterParams

filter = FilterParams.from_tuples(
    'name', 'ABC'
)
result = persistence.get_page_filter(None, filter, None)
Not available

In the persistence component, the developer is responsible for parsing FilterParams and passing a filter function to the persistence’s methods of the base class.


private composeFilter(filter: FilterParams): any {
    filter = filter || new FilterParams();

    let criteria = [];

    let id = filter.getAsNullableString('id');
    if (id != null) {
        criteria.push({ _id: id });
    }

    let siteId = filter.getAsNullableString('site_id');
    if (siteId != null) {
        criteria.push({ site_id: siteId });
    }

    let label = filter.getAsNullableString('label');
    if (label != null) {
        criteria.push({ label: label });
    }

    let udi = filter.getAsNullableString('udi');
    if (udi != null) {
        criteria.push({ udi: udi });
    }

    let udis = filter.getAsObject('udis');
    if (typeof udis === "string") {
        udis = udis.split(',');
    }
    if (Array.isArray(udis)) {
        criteria.push({ udi: { $in: udis } });
    }

    return criteria.length > 0 ? { $and: criteria } : null;
}






func (c *BeaconsMongoDbPersistence) composeFilter(filter *cquery.FilterParams) interface{} {
	if filter == nil {
		filter = cquery.NewEmptyFilterParams()
	}

	criteria := make([]bson.M, 0, 0)

	id := filter.GetAsString("id")
	if id != "" {
		criteria = append(criteria, bson.M{"_id": id})
	}

	siteId := filter.GetAsString("site_id")
	if siteId != "" {
		criteria = append(criteria, bson.M{"site_id": siteId})
	}
	label := filter.GetAsString("label")
	if label != "" {
		criteria = append(criteria, bson.M{"label": label})
	}
	udi := filter.GetAsString("udi")
	if udi != "" {
		criteria = append(criteria, bson.M{"udi": udi})
	}

	udis := filter.GetAsString("udis")
	var arrUdis []string = make([]string, 0, 0)
	if udis != "" {
		arrUdis = strings.Split(udis, ",")
		if len(arrUdis) > 1 {
			criteria = append(criteria, bson.M{"udi": bson.D{{"$in", arrUdis}}})
		}
	}
	if len(criteria) > 0 {
		return bson.D{{"$and", criteria}}
	}
	return bson.M{}
}



 def __compose_filter(self, filter: FilterParams) -> Callable:
        filter = filter if filter is not None else FilterParams()

        id = filter.get_as_nullable_string("id")
        site_id = filter.get_as_nullable_string("site_id")
        label = filter.get_as_nullable_string("label")
        udi = filter.get_as_nullable_string("udi")
        udis = filter.get_as_object("udis")
        if udis is not None and len(udis) > 0:
            udis = udis.split(",")

        def filter_beacons(item):
            if id is not None and item.id != id:
                return False
            if site_id is not None and item.site_id != site_id:
                return False
            if label is not None and item.label != label:
                return False
            if udi is not None and item.udi != udi:
                return False
            if udis is not None and item.udi not in udis:
                return False
            return True

        return filter_beacons
Not available

Paging

Another common data pattern is Paging. It is used to retrieve large datasets in chunks, through multiple calls to the storage. A client can ask for the results to be paged by specifying a set of PagingParams, which include the starting position and the number of objects to return. Clients can also request the total number of items in the dataset using PagingParams, but this parameter is optional. A DataPage object with a subset of the data will be returned as the result.

//skip = 25, take = 50, total = False
let paging = new PagingParams(25, 50, false);
let result = await persistence.getPageByFilter(null, null, paging);

See: PagingParams

// skip = 25, take = 50, total = False
paging := cquery.NewPagingParams(25, 50, false)
result := persistence.GetPageByFilter(context.Background(), nil, paging)

See: PagingParams

# skip = 25, take = 50, total = False
paging = PagingParams(25, 50, False)
result = persistence.get_page_by_filter(None, None, paging)
Not available

Custom Persistence Methods

As mentioned above, developers can also implement custom persistence methods. The _collection property can be used to access data objects from within such methods. Below is an example of a custom getOneByUdi persistence method.

public async getOneByUdi(correlationId: string, udi: string): Promise<BeaconV1> {
    let criteria = {
        udi: udi
    };
    return new Promise((resolve, reject) => {
        this._collection.findOne(criteria, (err, item) => {
            if (err != null) {
                reject(err);
                return;
            }
            if (item != null) this._logger.trace(correlationId, "Found beacon by %s", udi);
            else this._logger.trace(correlationId, "Cannot find beacon by %s", udi);
            
            item = this.convertToPublic(item);
            resolve(item);
        });    
    });     
}






func (c *BeaconsMongoPersistence) GetOneById(ctx context.Context, id string) (item data1.BeaconV1, err error) {
	return c.IdentifiableMongoDbPersistence.GetOneById(context.Background(), id)
}



 def get_one_by_udi(self, context: Optional[IContext], udi: Any) -> T:

    item = self._collection.find_one({'udi': udi})

    if item is None:
        self._logger.trace(context, "Nothing found from %s with udi = %s", self._collection_name, udi)
    else:
        self._logger.trace(context, "Retrieved from %s with udi = %s", self._collection_name, udi)

    item = self._convert_to_public(item)

    return item
Not available

When we put everything together, we end up with the following component:

export class BeaconsMongoDbPersistence
    extends IdentifiableMongoDbPersistence<BeaconV1, string>
    implements IBeaconsPersistence {

    constructor() {
        super('beacons');
        this._maxPageSize = 1000;
    }

    private composeFilter(filter: FilterParams): any {
        filter = filter || new FilterParams();

        let criteria = [];

        let id = filter.getAsNullableString('id');
        if (id != null) {
            criteria.push({ _id: id });
        }

        let siteId = filter.getAsNullableString('site_id');
        if (siteId != null) {
            criteria.push({ site_id: siteId });
        }

        let label = filter.getAsNullableString('label');
        if (label != null) {
            criteria.push({ label: label });
        }

        let udi = filter.getAsNullableString('udi');
        if (udi != null) {
            criteria.push({ udi: udi });
        }

        let udis = filter.getAsObject('udis');
        if (typeof udis === "string") {
            udis = udis.split(',');
        }
        if (Array.isArray(udis)) {
            criteria.push({ udi: { $in: udis } });
        }

        return criteria.length > 0 ? { $and: criteria } : null;
    }

    public async getPageByFilter(ctx: Context, filter: FilterParams,
        paging: PagingParams): Promise<DataPage<BeaconV1>> {
            DataPage.
        return super.getPageByFilter(ctx, this.composeFilter(filter), paging, null, null);
    }

    public async getOneByUdi(ctx: Context, udi: string): Promise<BeaconV1> {
        let criteria = {
            udi: udi
        };

        return new Promise((resolve, reject) => {
            this._collection.findOne(criteria, (err, item) => {
                if (err != null) {
                    reject(err);
                    return;
                }

                if (item != null) this._logger.trace(ctx, "Found beacon by %s", udi);
                else this._logger.trace(ctx, "Cannot find beacon by %s", udi);
                
                item = this.convertToPublic(item);
                resolve(item);
            });    
        });     
    }
}





import (
	"context"
	"strings"

	data1 "github.com/pip-services-samples/service-beacons-go/data/version1"
	cquery "github.com/pip-services4/pip-services4-go/pip-services4-data-go/query"
	cmongo "github.com/pip-services4/pip-services4-go/pip-services4-mongodb-go/persistence"
	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/mongo"
)

type BeaconsMongoDbPersistence struct {
	cmongo.IdentifiableMongoDbPersistence[data1.BeaconV1, string]
}

func NewBeaconsMongoDbPersistence() *BeaconsMongoDbPersistence {
	c := &BeaconsMongoDbPersistence{}
	c.IdentifiableMongoDbPersistence = *cmongo.InheritIdentifiableMongoDbPersistence[data1.BeaconV1, string](c, "beacons")
	return c
}

func (c *BeaconsMongoDbPersistence) composeFilter(filter *cquery.FilterParams) interface{} {
	if filter == nil {
		filter = cquery.NewEmptyFilterParams()
	}

	criteria := make([]bson.M, 0, 0)

	id := filter.GetAsString("id")
	if id != "" {
		criteria = append(criteria, bson.M{"_id": id})
	}

	siteId := filter.GetAsString("site_id")
	if siteId != "" {
		criteria = append(criteria, bson.M{"site_id": siteId})
	}
	label := filter.GetAsString("label")
	if label != "" {
		criteria = append(criteria, bson.M{"label": label})
	}
	udi := filter.GetAsString("udi")
	if udi != "" {
		criteria = append(criteria, bson.M{"udi": udi})
	}

	udis := filter.GetAsString("udis")
	var arrUdis []string = make([]string, 0, 0)
	if udis != "" {
		arrUdis = strings.Split(udis, ",")
		if len(arrUdis) > 1 {
			criteria = append(criteria, bson.M{"udi": bson.D{{"$in", arrUdis}}})
		}
	}
	if len(criteria) > 0 {
		return bson.D{{"$and", criteria}}
	}
	return bson.M{}
}

func (c *BeaconsMongoDbPersistence) GetPageByFilter(ctx context.Context, correlationId string, filter *cquery.FilterParams, paging cquery.PagingParams) (page cquery.DataPage[data1.BeaconV1], err error) {
	return c.IdentifiableMongoDbPersistence.GetPageByFilter(ctx, c.composeFilter(filter), paging, nil, nil)
}

func (c *BeaconsMongoDbPersistence) GetOneByUdi(ctx context.Context, udi string) (result data1.BeaconV1, err error) {
	filter := bson.M{"udi": udi}
	var docPointer map[string]any
	foRes := c.Collection.FindOne(ctx, filter)

	ferr := foRes.Decode(&docPointer)
	if ferr != nil {
		if ferr == mongo.ErrNoDocuments {
			return data1.BeaconV1{}, nil
		}
		return data1.BeaconV1{}, ferr
	}

	result, err = c.ConvertToPublic(docPointer)
	if err != nil {
		return result, err
	}
	return result, nil
}



class BeaconsMongoDbPersistence(IdentifiableMongoDbPersistence, IBeaconsPersistence):

    def __init__(self):
        super(BeaconsMongoDbPersistence, self).__init__("beacons")
        self._max_page_size = 1000

    def compose_filter(self, filter: FilterParams) -> Any:
        filter = filter if filter is not None else FilterParams()
        criteria = []

        id = filter.get_as_nullable_string("id")
        if id is not None:
            criteria.append({"id": id})
        site_id = filter.get_as_nullable_string("site_id")
        if site_id is not None:
            criteria.append({"site_id": site_id})
        label = filter.get_as_nullable_string("label")
        if label is not None:
            criteria.append({"label": label})
        udi = filter.get_as_nullable_string("udi")
        if udi is not None:
            criteria.append({"udi": udi})
        udis = filter.get_as_object("udis")
        if udis is not None and len(udis) > 0:
            udis = udis.split(",")
            criteria.append({"udi": {"$in": udis}})
        return {"$and": criteria} if len(criteria) > 0 else None

    def get_page_by_filter(self, context: IContext, filter: FilterParams, paging: PagingParams) -> T
        return get_page_by_filter(context, self.__compose_filter(filter), paging, None, None)

    def get_one_by_udi(self, context: Optional[IContext], udi: Any) -> T:

        item = self._collection.find_one({'udi': udi})

        if item is None:
            self._logger.trace(context, "Nothing found from %s with udi = %s", self._collection_name, udi)
        else:
            self._logger.trace(context, "Retrieved from %s with udi = %s", self._collection_name, udi)

        item = self._convert_to_public(item)

        return item
Not available

The following example demonstrates how we can use our newly created persistence for writing and reading Beacon objects to a MongoDB:

let persistence = new BeaconsMongoDbPersistence();

persistence.Configure(cconf.NewConfigParamsFromTuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test",
))

await persistence.open(null);

let beacon = <BeaconV1>{
    id: '1', 
    site_id: "0001",
    udi: "0002"
};

await persistence.set(ctx, beacon)
let item = await persistence.getOneByUdi(ctx, "0002")
console.log(item.udi) // Result: 0002
let page = await persistence.getPageByFilter(ctx, FilterParams.fromTuples("udi", "0002"), null)

console.log(page.data.length) // Result: 1
console.log(page.data[0].udi)   // Result: 0002
await persistence.close(ctx)





persistence := NewBeaconsMongoDbPersistence()

persistence.Configure(context.Background(), cconf.NewConfigParamsFromTuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test",
))

persistence.Open(context.Background())
beacon := &data1.BeaconV1{Id: "1", SiteId: "0001", Udi: "0002"}

persistence.Set(context.Background(), *beacon)
item, _ := persistence.GetOneByUdi(context.Background(), "0002")

fmt.Println(item.Udi) // Result: 0002

itemsPage, _ := persistence.GetPageByFilter(context.Background(), "test", cquery.NewFilterParamsFromTuples("udi", "0002"), cquery.PagingParams{})

fmt.Println(len(itemsPage.Data))   // Result: 1
fmt.Println(itemsPage.Data[0].Udi) // Result: 0002
persistence.Close(context.Background())



persistence = BeaconsMongoDbPersistence()

persistence.configure(ConfigParams.from_tuples(
	"connection.host", "localhost",
	"connection.port", "30000",
	"connection.database", "test"
))

persistence.open(None)
beacon = BeaconV1(id="1", site_id="0001", udi="0002")

persistence.set("test", beacon)
item = persistence.get_one_by_udi("test", "0002")

print(item.udi)   # Result: 0002

items_page = persistence.get_page_by_filter("test", FilterParams.from_tuples("udi", "0002"), None)

print(len(items_page.data)) # Result: 1
print(items_page.data[0].udi)   # Result: 0002
persistence.close("test")
Not available