Persistence

Persisting data is one of the most common functions in backend services. There are many good persistence frameworks available on the market and the Pip.Services toolkit doesn’t intend to compete with them. Using the Component Model the toolkit integrates selected best of breed persistence technologies to enable consistent configuration, management and the use of the symmetric programming model across different databases and languages.

Out of the box, the Pip.Services toolkit offers support for a number of persistence technologies and popular relational and NoSQL databases (implementations for different languages may vary). They are:

  • In-memory persistence
  • File persistence
  • SQL Server
  • MySQL
  • Postresql
  • SQLite
  • MongoDB
  • Couchbase
  • Cassandra

Connection Management

The XyzConnection component encapsulates database connections and configures them like on the example config file below:

# Database connection
descriptor: pip-services:connection:mongodb:default:3.0
connection:
  host: ${{MONGO_HOST}}{$unless MONGO_HOST}localhost{$unless}
  port: ${{MONGO_PORT}}{$unless MONGO_PORT}27017{$unless}
credentials:
  host: ${{MONGO_USER}}{$unless MONGO_USER}mongo{$unless}
  port: ${{MONGO_PASS}}{$unless MONGO_PASS}pwd123{$unless}
options:
  max_pool_size: 10
  keep_alive: true

Using the connection.discovery_key parameter, the connection component can retrieve connection parameters from Discovery Services and, using the credentials.store_key parameter, it can retrieve credential parameters from Credential Stores.

By default, persistence components will try to retrieve the first available connection from the references. By specifying the references.connection parameter, a persistence component can be linked with a specific connection. If there are no connections available, each persistence component will try to create its own connection. See the sample config below:

# Persistence with default connection
descriptor: myservice:mypersistence:mongodb:persist1:1.0

# Persistence linked to specific connection
descriptor: myservice:mypersistence:mongodb:persist2:1.0
references:
  connection: pip-services:connection:mongodb:conn1:3.0

Most XyzConnection components have the getConnection() method (Connection property) to get a reference to a shared database connection.

Generic Persistence

For persistence operations against a single table (collection) the toolkit offers XyzPersistence components. They act as abstract classes for specific implementations that enable the component lifecycle and share database connections. Persistence operations can be implemented using custom code or a number of out-of-the-box methods like GetListByFilter(), Create() or DeleteByFilter().

import { Context } from "pip-services4-components-node";
import { MongoDbPersistence } from 'pip-services4-mongodb-node';

class MyObject {
  key: string;
  name: string;
}

class MyMongoDbPersistence extends MongoDbPersistence<MyObject> {

  public constructor() {
    super("mycollection");
  }

  public async getByName(ctx: Context, name: string): Promise<MyObject> {
    let criteria= { name: name };
    let res = await super.getListByFilter(ctx, criteria, null, null);
    return res.length > 0 ? res[0] : null;
  }

  public async createDefault(ctx: Context, name: string): Promise<MyObject> {
    name = name ?? "unknown";
    let key = name.toLowerCase().replace(" #$%^&", "_");
    let item: MyObject = { key: key, name: name };

      let result = await new Promise<any>((resolve, reject) => {
      this._collection.insertOne(item, (err, result) => {
        if (err = null) resolve(result);
        else reject(err);
      });
    });

    this._logger.trace(ctx, "Created item in %s with key = %s", this._collectionName, key);

   let newItem = result != null ? result.ops[0] : null;
   return newItem;
  }

  public async deleteByName(ctx: Context, name: string): Promise<void> {
    let criteria= { name: name };
    await super.deleteByFilter(ctx, criteria);
  }
}



import (
	"context"
	"strings"

	persist "github.com/pip-services4/pip-services4-go/pip-services4-mongodb-go/persistence"
	"go.mongodb.org/mongo-driver/bson"
)

type MyObject struct {
	Key  string `bson:"key" json:"key"`
	Name string `bson:"name" json:"name"`
}

type MyMongoDbPersistence struct {
	*persist.MongoDbPersistence[MyObject]
}

func NewMyMongoDbPersistence() *MyMongoDbPersistence {
	c := &MyMongoDbPersistence{}
	c.MongoDbPersistence = persist.InheritMongoDbPersistence[MyObject](c, "mycollection")
	return c
}

func (c *MyMongoDbPersistence) GetByName(ctx context.Context, name string) (result MyObject, err error) {

	filterObj := bson.M{"name": name}

	items, err := c.MongoDbPersistence.GetListByFilter(ctx, filterObj, nil, nil)
	if err != nil {
		return result, err
	}

	if len(items) > 0 {
		return items[0], nil
	} else {
		return result, nil
	}
}

func (c *MyMongoDbPersistence) CreateDefault(ctx context.Context, correlationId string,
	name string) (result MyObject, err error) {

	if name == "" {
		name = "unknown"
	}

	key := strings.ReplaceAll(strings.ToLower(name), " #$%^&", "_")
	item := MyObject{Key: key, Name: name}

	newItem, err := c.Overrides.ConvertFromPublic(item)
	if err != nil {
		return result, err
	}
	insRes, err := c.Collection.InsertOne(ctx, newItem)
	if err != nil {
		return result, err
	}

	result, err = c.Overrides.ConvertToPublic(newItem)
	if err != nil {
		return result, err
	}
	c.Logger.Trace(ctx, correlationId, "Created in %s with id = %s", c.Collection, insRes.InsertedID)
	return result, nil
}

func (c *MyMongoDbPersistence) DeleteByName(ctx context.Context, name string) error {
	filterObj := bson.M{"name": name}
	return c.DeleteByFilter(ctx, filterObj)
}




from pip_services4_mongodb.persistence import MongoDbPersistence


class MyObject:
    def __init__(self, key: str = None, name: str = None):
        self.name = key
        self.content = name


class MyMongoDbPersistence(MongoDbPersistence):

    def __init__(self):
        super(MyMongoDbPersistence, self).__init__('mycollection')

    def get_by_name(self, correlation_id: str, name: str) -> MyObject:
        criteria = {'name': name}
        res = self.get_list_by_filter(correlation_id, criteria, None, None)
        return None if len(res) < 0 else res[0]

    def create_default(self, correlation_id: str, name: str) -> MyObject:
        name = name or 'unknown'
        key = name.lower().replace(" #$%^&", "_")
        item = MyObject(key, name)

        result = self._collection.insert_one(item)
        item = self._collection.find_one({'_id': result.inserted_id})

        item = self._convert_to_public(item)
        return item

    def delete_by_name(self, correlation_id: str, name: str):
        criteria = {'name': name}
        self.delete_by_filter(correlation_id, criteria)
Not available

To implement custom persistence operations, protected properties should be used to get access to database connection, transaction and table (collection) object references encapsulated by the component (check the component documentation for details).

Identifiable Persistence

The most common persistence scenario is when data objects are identified by a unique Id field and stored in a single table (collection). This allows to implement persistence that can work against any relational or NoSQL persistence store available now or in the future (see concept Long-Living Code). To support this scenario the Pip.Services toolkit has the XyzIdentifiablePersistence components that offer a full set of CRUD operations for objects that have the Id property and implement the IIdentifiable interface.

import { IIdentifiable } from 'pip-services4-data-node';

class MyIdentifiableObject implements IIdentifiable <string> {
    public id: string;
    public name: string;
    public value: string;
}




type MyIdentifiableObject struct {
	Id    string `bson:"id" json:"id"`
	Name  string `bson:"name" json:"name"`
	Value string `bson:"value" json:"value"`
}

func (c *MyIdentifiableObject) GetId() string {
	return c.Id
}



from pip_services4_data.data import IIdentifiable

class MyIdentifiableObject(IIdentifiable):
    def __init__(self, id: str = None, name: str = None, value: str = None):
        self.id = id
        self.name = name
        self.value = value
Not available

Unique ids can be natural or generated by a special key generator. However, to make it simpler and portable, the toolkit has the IdGenerator class that can generate globally unique Ids as string GUIDs. Those ids are quite long (32 characters) but they work well for collections of moderate size.

The example below requires only a few lines of code, but implements a persistence with a full set of CRUD operations:

import { IIdentifiable, FilterParams, PagingParams, DataPage } from "pip-services4-data-node";
import { Context } from "pip-services4-components-node";
import { IdentifiablePostgresPersistence } from 'pip-services4-postgres-node';

class MyIdentifiableObject implements IIdentifiable <string> {
    public id: string;
    public name: string;
    public value: string;
}

interface MyIdentifiablePersistence {
    getPageByFilter(correlationId: string, filter: FilterParams, paging: PagingParams): Promise<DataPage<MyIdentifiableObject>>;
    create(correlationId: string, item: MyIdentifiableObject): Promise<MyIdentifiableObject>;
    getOneById(correlationId: string, id: string): Promise<MyIdentifiableObject>;
    deleteById(correlationId: string, id: string): Promise<MyIdentifiableObject>;
}

class MyIdentifiablePostgreSqlPersistence extends IdentifiablePostgresPersistence<MyIdentifiableObject, string> implements MyIdentifiablePersistence {

    public constructor() {
        super("mycollection");
    }

    public composeFilter(filter: FilterParams): string {
        filter = filter ?? new FilterParams();
        let criteria = [];

        let id = filter.getAsString("id");
        if (id != null) {
            criteria.push("id='" + id + "'");
        }

        let name = filter.getAsString("name");
        if (name != null) {
            criteria.push("name='" + name + "'");
        }

        return criteria.length > 0 ? criteria.join(" AND ") : null;
    }

    public async getPageByFilter(ctx: Context, filter: FilterParams, paging: PagingParams): Promise<DataPage<MyIdentifiableObject>> {
        let criteria = this.composeFilter(filter);
        return await super.getPageByFilter(ctx, criteria, paging, null, null);
    }
}




import (
	"context"
	"strings"

	cquery "github.com/pip-services4/pip-services4-go/pip-services4-data-go/query"
	persist "github.com/pip-services4/pip-services4-go/pip-services4-postgres-go/persistence"
)

type MyIdentifiableObject struct {
	Id    string `bson:"id" json:"id"`
	Name  string `bson:"name" json:"name"`
	Value string `bson:"value" json:"value"`
}

func (c *MyIdentifiableObject) GetId() string {
	return c.Id
}

type MyIdentifiablePersistence interface {
	GetPageByFilter(correlationId string, filter cquery.FilterParams, paging cquery.PagingParams) (cquery.DataPage[MyIdentifiableObject], error)
	Create(correlationId string, item MyIdentifiableObject) (MyIdentifiableObject, error)
	GetOneById(correlationId string, id string) (MyIdentifiableObject, error)
	DeleteById(correlationId string, id string) (MyIdentifiableObject, error)
}

type MyIdentifiablePostgreSqlPersistence struct {
	*persist.IdentifiablePostgresPersistence[MyIdentifiableObject, string]
}

func NewMyIdentifiablePostgreSqlPersistence() *MyIdentifiablePostgreSqlPersistence {
	c := &MyIdentifiablePostgreSqlPersistence{}
	c.IdentifiablePostgresPersistence = persist.InheritIdentifiablePostgresPersistence[MyIdentifiableObject, string](c, "mycollection")
	return c
}

func (c *MyIdentifiablePostgreSqlPersistence) composeFilter(filter cquery.FilterParams) string {
	criteria := make([]string, 0)

	if id, ok := filter.GetAsNullableString("id"); ok && id != "" {
		criteria = append(criteria, "id='"+id+"'")
	}

	if name, ok := filter.GetAsNullableString("name"); ok && name != "" {
		criteria = append(criteria, "name='"+name+"'")
	}

	if len(criteria) > 0 {
		return strings.Join(criteria, " AND ")
	} else {
		return ""
	}
}

func (c *MyIdentifiablePostgreSqlPersistence) GetPageByFilter(ctx context.Context,
	filter cquery.FilterParams, paging cquery.PagingParams) (page cquery.DataPage[MyIdentifiableObject], err error) {

	return c.IdentifiablePostgresPersistence.GetPageByFilter(ctx,
		c.composeFilter(filter), paging,
		"", "",
	)
}



from abc import ABC

from pip_services4_data.data import IIdentifiable
from pip_services4_postgres.persistence import IdentifiablePostgresPersistence
from pip_services4_data.query import FilterParams, PagingParams, DataPage


class MyIdentifiableObject(IIdentifiable):
    def __init__(self, id: str = None, name: str = None, value: str = None):
        self.id = id
        self.name = name
        self.value = value


class MyIdentifiablePersistence(ABC):
    def get_page_by_filter(self, correlation_id: str, filter: FilterParams, paging: PagingParams) -> DataPage:
        pass

    def create(self, correlation_id: str, item: MyIdentifiableObject) -> MyIdentifiableObject:
        pass

    def get_one_by_id(self, correlation_id: str, id: str) -> MyIdentifiableObject:
        pass

    def delete_by_id(self, correlation_id: str, id: str) -> MyIdentifiableObject:
        pass


class MyIdentifiablePostgreSqlPersistence(IdentifiablePostgresPersistence, MyIdentifiablePersistence):
    def __init__(self):
        super(MyIdentifiablePostgreSqlPersistence, self).__init__('mycollection')

    def _compose_filter(self, filter_params: FilterParams):
        filter_params = filter_params or FilterParams()
        criteria = []

        id = filter_params.get_as_string("id")
        if id is not None and id != "":
            criteria.append("id='" + id + "'")

        name = filter_params.get_as_string("name")
        if name is not None and name != "":
            criteria.append("name='" + name + "'")

        return None if len(criteria) < 0 else " AND ".join(criteria)

    def get_page_by_filter(self, correlation_id: str, filter: FilterParams, paging: PagingParams) -> DataPage:
        criteria = self._compose_filter(filter)
        return super(MyIdentifiablePostgreSqlPersistence, self).get_page_by_filter(correlation_id, criteria, paging, None, None)
Not available

Most relational XyzIdentifiablePersistence components have variations called XyzJsonIdentifiablePersistence that store objects in JSON format. In this case, tables have only 2 fields: id and data. The id field contains the unique object Id and the data field contains the entire object data serialized as JSON:

CREATE TABLE MyIdentifiableJsonObject (
  id VARCHAR(32) PRIMARY KEY,
  data JSON
);

For more information, please, refer to the component documentation.

Custom Persistence

There are situations, when persistence has a complex data model, or there is already an existing implementation that does the work. In those cases, it is only necessary to use that implementation written in a chosen persistence framework and wrap it into a Pip.Services component to enable configuration and life-cycle management in order to place that persistence into a container and connect to other components in a data service. Here is a simple example of how it can be done.

import { ConfigParams, Context, IConfigurable, IOpenable, IReferenceable, IReferences } from "pip-services4-components-node";

class MyCustomPersistence {
    // Custom implementation using any persistence framework
}

class MyCustomPersistenceWrapper implements IConfigurable, IReferenceable, IOpenable {
    public _config: ConfigParams = new ConfigParams();
    public _persistence: MyCustomPersistence;

    public configure(config: ConfigParams): void {
        // Store config parameters
        this._config = config ?? this._config;
    }

    public setReferences(references: IReferences): void {
        // Retrieve whatever references you may need
    }

    public isOpen(): boolean {
        return this._persistence != null;
    }

    public async open(ctx: Context): Promise<void> {
        if (this._persistence != null) return;

        // Create custom persistence
        this._persistence = new MyCustomPersistence();

        // Configure custom persistence
        ...

        // Open and connect to the database
        await this._persistence.connect();
    }

    public async close(ctx: Context): Promise<void> {
        if (this._persistence == null) return;

        // Disconnect from the database and close
        await this._persistence.disconnect();
        this._persistence = null;
    }

    public customMethod(...) {
        // Delegate operations to custom persistence
        return await this._persistence.customMethod(...);
    }
}



import (
	"context"

	cconf "github.com/pip-services4/pip-services4-go/pip-services4-components-go/config"
	cref "github.com/pip-services4/pip-services4-go/pip-services4-components-go/refer"
)

type MyCustomPersistence struct {
	// Custom implementation using any persistence framework
}

func NewMyCustomPersistence() *MyCustomPersistence {
	return &MyCustomPersistence{}
}

type MyCustomPersistenceWrapper struct {
	config      *cconf.ConfigParams
	persistence *MyCustomPersistence
}

func NewMyCustomPersistenceWrapper() *MyCustomPersistenceWrapper {
	return &MyCustomPersistenceWrapper{
		config: cconf.NewConfigParams(map[string]string{}),
	}
}

func (c *MyCustomPersistenceWrapper) Configure(ctx context.Context, config *cconf.ConfigParams) {
	// Store config parameters
	if config != nil {
		c.config = config
	}
}

func (c *MyCustomPersistenceWrapper) SetReferences(ctx context.Context, references cref.IReferences) {
	// Retrieve whatever references you may need
}

func (c *MyCustomPersistenceWrapper) IsOpen() bool {
	return c.persistence != nil
}

func (c *MyCustomPersistenceWrapper) Open(ctx context.Context, correlationId string) (err error) {
	if c.persistence != nil {
		return nil
	}

	// Create custom persistence
	c.persistence = NewMyCustomPersistence()

	// Configure custom persistence
	// ...

	// Open and connect to the database
	c.persistence.Connect()
}

func (c *MyCustomPersistenceWrapper) Close(ctx context.Context, correlationId string) (err error) {
	if c.persistence == nil {
		return
	}

	// Disconnect from the database and close
	c.persistence.Disconnect()
	c.persistence = nil
}




from typing import Optional

from pip_services4_components.config import IConfigurable, ConfigParams
from pip_services4_components.refer import IReferenceable, IReferences
from pip_services4_components.run import IOpenable


class MyCustomPersistence:
    # Custom implementation using any persistence framework
    pass

class MyCustomPersistenceWrapper(IConfigurable, IReferenceable, IOpenable):

    def __init__(self):
        self._config = ConfigParams()
        self._persistence: MyCustomPersistence = None

    def configure(self, config: ConfigParams):
        # Store config parameters
        self._config = config or self._config

    def set_references(self, references: IReferences):
        # Retrieve whatever references you may need
        pass

    def is_open(self) -> bool:
        return self._persistence is not None

    def open(self, correlation_id: Optional[str]):
        if self._persistence is not None:
            return

        # Create custom persistence
        self._persistence = MyCustomPersistence()

        # Configure custom persistence
        # ...

        # Open and connect to the database
        self._persistence.connect()

    def close(self, correlation_id: Optional[str]):
        if self._persistence is None:
            return

        # Disconnect from the database and close
        self._persistence.disconnect()
        self._persistence = None

    def custom_method(self, ...):
        # Delegate operations to custom persistence
        return self._persistence.custom_method(...)
Not available

References

For more information about configurations see: