needlestack.apis package

Submodules

needlestack.apis.collections_pb2 module

class needlestack.apis.collections_pb2.Collection

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

enable_id_to_vector

Field Collection.enable_id_to_vector

name

Field Collection.name

replication_factor

Field Collection.replication_factor

shards

Field Collection.shards

class needlestack.apis.collections_pb2.CollectionsAddRequest

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

balance_algorithm

Field CollectionsAddRequest.balance_algorithm

collections

Field CollectionsAddRequest.collections

noop

Field CollectionsAddRequest.noop

class needlestack.apis.collections_pb2.CollectionsAddResponse

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

collections

Field CollectionsAddResponse.collections

success

Field CollectionsAddResponse.success

class needlestack.apis.collections_pb2.CollectionsDeleteRequest

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

names

Field CollectionsDeleteRequest.names

noop

Field CollectionsDeleteRequest.noop

class needlestack.apis.collections_pb2.CollectionsDeleteResponse

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

names

Field CollectionsDeleteResponse.names

success

Field CollectionsDeleteResponse.success

class needlestack.apis.collections_pb2.CollectionsListRequest

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

names

Field CollectionsListRequest.names

class needlestack.apis.collections_pb2.CollectionsListResponse

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

collections

Field CollectionsListResponse.collections

class needlestack.apis.collections_pb2.CollectionsLoadRequest

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

class needlestack.apis.collections_pb2.CollectionsLoadResponse

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

success

Field CollectionsLoadResponse.success

class needlestack.apis.collections_pb2.Node

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

hostname

Field Node.hostname

hostport

Field Node.hostport

ip_address

Field Node.ip_address

port

Field Node.port

class needlestack.apis.collections_pb2.Replica

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

ACTIVE = 1

BOOTING = 3

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

DOWN = 2

RECOVERING = 4

State = <google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper object>

UNKNOWN = 0

collection_name

Field Replica.collection_name

leader

Field Replica.leader

node

Field Replica.node

shard_name

Field Replica.shard_name

state

Field Replica.state

class needlestack.apis.collections_pb2.Shard

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

collection_name

Field Shard.collection_name

index

Field Shard.index

name

Field Shard.name

replicas

Field Shard.replicas

weight

Field Shard.weight

needlestack.apis.collections_pb2_grpc module

needlestack.apis.data_sources_pb2 module

class needlestack.apis.data_sources_pb2.DataSource

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

gcs_data_source

Field DataSource.gcs_data_source

local_data_source

Field DataSource.local_data_source

class needlestack.apis.data_sources_pb2.GcsDataSource

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

blob_name

Field GcsDataSource.blob_name

bucket_name

Field GcsDataSource.bucket_name

credentials_file

Field GcsDataSource.credentials_file

project_name

Field GcsDataSource.project_name

class needlestack.apis.data_sources_pb2.LocalDataSource

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

filename

Field LocalDataSource.filename

needlestack.apis.data_sources_pb2_grpc module

needlestack.apis.indexing module

needlestack.apis.indexing.create_faiss_index_shard(faiss_index, metadatas)

A simple function to that takes a faiss index object and metadata for vectors to create a serializable FaissIndex object. It would have been nice to be able to pass in vectors and metadatas, but creating a faiss index requires developers make decisions about hyper-parameters that would be difficult to automate.

Return type

BaseIndex

needlestack.apis.indices_pb2 module

class needlestack.apis.indices_pb2.BaseIndex

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

faiss_index

Field BaseIndex.faiss_index

class needlestack.apis.indices_pb2.FaissIndex

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

data_source

Field FaissIndex.data_source

index_binary

Field FaissIndex.index_binary

metadatas

Field FaissIndex.metadatas

class needlestack.apis.indices_pb2.Metadata

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

fields

Field Metadata.fields

id

Field Metadata.id

class needlestack.apis.indices_pb2.MetadataField

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

bool_val

Field MetadataField.bool_val

double_val

Field MetadataField.double_val

float_val

Field MetadataField.float_val

int_val

Field MetadataField.int_val

long_val

Field MetadataField.long_val

name

Field MetadataField.name

string_val

Field MetadataField.string_val

class needlestack.apis.indices_pb2.RetrievalResultItem

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

metadata

Field RetrievalResultItem.metadata

vector

Field RetrievalResultItem.vector

class needlestack.apis.indices_pb2.SearchResultItem

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

double_distance

Field SearchResultItem.double_distance

float_distance

Field SearchResultItem.float_distance

metadata

Field SearchResultItem.metadata

needlestack.apis.indices_pb2_grpc module

needlestack.apis.serializers module

needlestack.apis.serializers.metadata_field_to_proto(field, fieldtype=None, fieldname=None)

Serialize some python value to a metadata field proto

Parameters

• field (Union[str, int, float, bool]) – Primative python value

• fieldtype (Optional[str]) – Explicit type to serialize the field

• fieldname (Optional[str]) – Optional name for this metadata field

Return type

MetadataField

needlestack.apis.serializers.metadata_list_to_proto(ids, fields_list, fieldtypes=None, fieldnames=None)

Serialize a set of items with metadata fields

Parameters

• ids (List[str]) – List of ids for items

• fields_list (List[Tuple]) – List of tuple of field values

• fieldtypes (Optional[Tuple[str]]) – Optional tuple of types for values

• fieldname – Optional tuple of names for values

Return type

List[Metadata]

needlestack.apis.serializers.metadata_to_proto(id, fields, fieldtypes=None, fieldnames=None)

Serialize a set of metadata fields for some item. Skips over None fields

Parameters

• id (str) – ID for item

• fields (Tuple) – Tuple of primative python values

• fieldtypes (Optional[Tuple[str]]) – Optional tuple of types for values

• fieldnames (Optional[Tuple[str]]) – Optional tuple of names for values

Return type

Metadata

needlestack.apis.serializers.ndarray_to_proto(X, dtype=None, shape=None)

Transforms a Python n-dimension array into a protobuf

Parameters

• X (Any) – ND Array

• dtype (Optional[str]) – Explicit datatype for number

• shape (Optional[Tuple]) – Explicit shape for nd array

Return type

NDArray

needlestack.apis.serializers.proto_to_ndarray(proto)

Transform a protobuf into a numpy array

Parameters

proto (NDArray) – Protobuf for nd array

Return type

ndarray

needlestack.apis.servicers_pb2 module

class needlestack.apis.servicers_pb2.RetrieveRequest

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

collection_name

Field RetrieveRequest.collection_name

id

Field RetrieveRequest.id

shard_names

Field RetrieveRequest.shard_names

class needlestack.apis.servicers_pb2.RetrieveResponse

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

item

Field RetrieveResponse.item

class needlestack.apis.servicers_pb2.SearchRequest

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

collection_name

Field SearchRequest.collection_name

count

Field SearchRequest.count

shard_names

Field SearchRequest.shard_names

vector

Field SearchRequest.vector

class needlestack.apis.servicers_pb2.SearchResponse

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

items

Field SearchResponse.items

needlestack.apis.servicers_pb2_grpc module

class needlestack.apis.servicers_pb2_grpc.MergerServicer

Bases: object

Main entrypoint for external clients to run distributed kNN searches

CollectionsAdd(request, context)

CollectionsDelete(request, context)

CollectionsList(request, context)

CollectionsLoad(request, context)

Retrieve(request, context)

Search(request, context)

SearchStream(request, context)

class needlestack.apis.servicers_pb2_grpc.MergerStub(channel)

Bases: object

Main entrypoint for external clients to run distributed kNN searches

class needlestack.apis.servicers_pb2_grpc.SearcherServicer

Bases: object

Worker used by Merger to perform single-node kNN search

CollectionsLoad(request, context)

Retrieve(request, context)

Search(request, context)

SearchStream(request, context)

class needlestack.apis.servicers_pb2_grpc.SearcherStub(channel)

Bases: object

Worker used by Merger to perform single-node kNN search

needlestack.apis.servicers_pb2_grpc.add_MergerServicer_to_server(servicer, server)

needlestack.apis.servicers_pb2_grpc.add_SearcherServicer_to_server(servicer, server)

needlestack.apis.tensors_pb2 module

class needlestack.apis.tensors_pb2.NDArray

Bases: google.protobuf.pyext._message.CMessage, google.protobuf.message.Message

DESCRIPTOR = <google.protobuf.pyext._message.MessageDescriptor object>

DType = <google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper object>

FLOAT16 = 1

FLOAT32 = 2

FLOAT64 = 3

INT16 = 5

INT32 = 6

INT64 = 7

INT8 = 4

UNKNOWN = 0

double_val

Field NDArray.double_val

dtype

Field NDArray.dtype

float_val

Field NDArray.float_val

int_val

Field NDArray.int_val

long_val

Field NDArray.long_val

numpy_content

Field NDArray.numpy_content

shape

Field NDArray.shape

needlestack.apis.tensors_pb2_grpc module

Module contents