SourceSync.ai MCP Server

OverviewInspectSchema Related Servers Reviews Score
export type SourceSyncApiResponse<T> = {
  success: boolean
  message: string
  data: T
  error?: {
    code: string
    message: string
    details?: { reason: string } & Record<string, any>
  }
}

export type SourceSyncFaunaTime = {
  isoString: string
}

export type SourceSyncFaunaRef = {
  id: string
}

export type SourceSyncNamespaceRef = {
  identifier: string
}

export type SourceSyncPaginationRequest = {
  pageSize?: number
  cursor?: string
}

export type SourceSyncPaginationResponse = {
  itemsReturned: number
  hasNextPage: boolean
  nextCursor: string | null
}

export type SourceSyncPaginatedResult<T> = {
  data: T[]
  hasNextPage: boolean
  nextCursor: string | null
}

export type SourceSyncDocumentsPaginatedResult =
  SourceSyncPaginatedResult<SourceSyncDocument> & {
    statsBySource: { source: SourceSyncIngestionSource; totalCount: number }[]
    statsByStatus: { status: SourceSyncIngestionStatus; totalCount: number }[]
  }

// ************************************************************************* //
// ***************************** ⬇⬇ ENUMS ⬇⬇ **************************** //
// ************************************************************************* //

export enum SourceSyncWebScraperProvider {
  FIRECRAWL = 'FIRECRAWL',
  JINA = 'JINA',
  SCRAPINGBEE = 'SCRAPINGBEE',
}

export enum SourceSyncFileStorageType {
  S3_COMPATIBLE = 'S3_COMPATIBLE',
}

export enum SourceSyncVectorStorageProvider {
  PINECONE = 'PINECONE',
}

export enum SourceSyncEmbeddingModelProvider {
  OPENAI = 'OPENAI',
  COHERE = 'COHERE',
  JINA = 'JINA',
}

export enum SourceSyncOpenAIEmbeddingModel {
  TEXT_EMBEDDING_3_SMALL = 'text-embedding-3-small',
  TEXT_EMBEDDING_3_LARGE = 'text-embedding-3-large',
  TEXT_EMBEDDING_ADA_002 = 'text-embedding-ada-002',
}

export enum SourceSyncCohereEmbeddingModel {
  EMBED_ENGLISH_V3 = 'embed-english-v3.0',
  EMBED_MULTILINGUAL_V3 = 'embed-multilingual-v3.0',
  EMBED_ENGLISH_LIGHT_V3 = 'embed-english-light-v3.0',
  EMBED_MULTILINGUAL_LIGHT_V3 = 'embed-multilingual-light-v3.0',
  EMBED_ENGLISH_V2 = 'embed-english-v2.0',
  EMBED_ENGLISH_LIGHT_V2 = 'embed-english-light-v2.0',
  EMBED_MULTILINGUAL_V2 = 'embed-multilingual-v2.0',
}

export enum SourceSyncJinaEmbeddingModel {
  JINA_EMBEDDINGS_V3 = 'jina-embeddings-v3',
}

export enum SourceSyncConnector {
  NOTION = 'NOTION',
  GOOGLE_DRIVE = 'GOOGLE_DRIVE',
  DROPBOX = 'DROPBOX',
  ONEDRIVE = 'ONEDRIVE',
  BOX = 'BOX',
  SHAREPOINT = 'SHAREPOINT',
}

export enum SourceSyncConnectionStatus {
  PENDING = 'PENDING',
  ACTIVE = 'ACTIVE',
  FAILED = 'FAILED',
  REVOKED = 'REVOKED',
}

export enum SourceSyncConnectionAuthType {
  OAUTH2 = 'OAUTH2',
}

export enum SourceSyncIngestionSource {
  TEXT = 'TEXT',
  URLS_LIST = 'URLS_LIST',
  SITEMAP = 'SITEMAP',
  WEBSITE = 'WEBSITE',
  LOCAL_FILE = 'LOCAL_FILE',
  NOTION = 'NOTION',
  GOOGLE_DRIVE = 'GOOGLE_DRIVE',
  DROPBOX = 'DROPBOX',
  ONEDRIVE = 'ONEDRIVE',
  BOX = 'BOX',
  SHAREPOINT = 'SHAREPOINT',
}

export enum SourceSyncIngestionStatus {
  BACKLOG = 'BACKLOG',
  QUEUED = 'QUEUED',
  QUEUED_FOR_RESYNC = 'QUEUED_FOR_RESYNC',
  PROCESSING = 'PROCESSING',
  SUCCESS = 'SUCCESS',
  FAILED = 'FAILED',
  CANCELLED = 'CANCELLED',
}

enum SourceSyncIngestJobRunStatus {
  QUEUED = 'QUEUED',
  PRE_PROCESSING = 'PRE_PROCESSING',
  PROCESSING = 'PROCESSING',
  COMPLETED = 'COMPLETED',
}

export enum SourceSyncDocumentType {
  TEXT = 'TEXT',
  URL = 'URL',
  FILE = 'FILE',
  NOTION_DOCUMENT = 'NOTION_DOCUMENT',
  GOOGLE_DRIVE_DOCUMENT = 'GOOGLE_DRIVE_DOCUMENT',
  DROPBOX_DOCUMENT = 'DROPBOX_DOCUMENT',
  ONEDRIVE_DOCUMENT = 'ONEDRIVE_DOCUMENT',
  BOX_DOCUMENT = 'BOX_DOCUMENT',
  SHAREPOINT_DOCUMENT = 'SHAREPOINT_DOCUMENT',
}

export enum SourceSyncSearchType {
  SEMANTIC = 'SEMANTIC',
  HYBRID = 'HYBRID',
}

// ************************************************************************* //
// ***************************** ⬆⬆ ENUMS ⬆⬆ **************************** //
// ************************************************************************* //

// ************************************************************************* //
// ************************** ⬇⬇ NAMESPACES ⬇⬇ ************************** //
// ************************************************************************* //

export type SourceSyncFileStorageConfig = {
  type: SourceSyncFileStorageType.S3_COMPATIBLE
  bucket: string
  region: string
  endpoint: string
  credentials: {
    accessKeyId: string
    secretAccessKey: string
  }
}

export type SourceSyncVectorStorageConfig = {
  provider: SourceSyncVectorStorageProvider.PINECONE
  apiKey: string
  indexHost: string
}

export type SourceSyncEmbeddingModelConfig =
  | {
      provider: SourceSyncEmbeddingModelProvider.OPENAI
      model: SourceSyncOpenAIEmbeddingModel
      apiKey: string
    }
  | {
      provider: SourceSyncEmbeddingModelProvider.COHERE
      model: SourceSyncCohereEmbeddingModel
      apiKey: string
    }
  | {
      provider: SourceSyncEmbeddingModelProvider.JINA
      model: SourceSyncJinaEmbeddingModel
      apiKey: string
    }

export type SourceSyncWebScraperConfig = {
  provider:
    | SourceSyncWebScraperProvider.FIRECRAWL
    | SourceSyncWebScraperProvider.JINA
    | SourceSyncWebScraperProvider.SCRAPINGBEE
  apiKey: string
}

export type SourceSyncNotionConfig = {
  clientId: string
  clientSecret: string
}

export type SourceSyncGoogleDriveConfig = {
  clientId: string
  clientSecret: string
  apiKey: string
}

export type SourceSyncDropboxConfig = {
  clientId: string
  clientSecret: string
}

export type SourceSyncOnedriveConfig = {
  clientId: string
  clientSecret: string
}

export type SourceSyncBoxConfig = {
  clientId: string
  clientSecret: string
}

export type SourceSyncSharepointConfig = {
  clientId: string
  clientSecret: string
}

export type SourceSyncNamespace = {
  identifier: string
  name: string
  organization: SourceSyncFaunaRef
  fileStorageConfig: SourceSyncFileStorageConfig
  vectorStorageConfig: SourceSyncVectorStorageConfig
  embeddingModelConfig: SourceSyncEmbeddingModelConfig
  webScraperConfig: SourceSyncWebScraperConfig | null
  notionConfig: SourceSyncNotionConfig | null
  googleDriveConfig: SourceSyncGoogleDriveConfig | null
  dropboxConfig: SourceSyncDropboxConfig | null
  onedriveConfig: SourceSyncOnedriveConfig | null
  boxConfig: SourceSyncBoxConfig | null
  sharepointConfig: SourceSyncSharepointConfig | null
  createdAt: SourceSyncFaunaTime
  updatedAt: SourceSyncFaunaTime
}

export type SourceSyncCreateNamespaceRequest = {
  name: string
  fileStorageConfig: SourceSyncFileStorageConfig
  vectorStorageConfig: SourceSyncVectorStorageConfig
  embeddingModelConfig: SourceSyncEmbeddingModelConfig
  webScraperConfig?: SourceSyncWebScraperConfig
}

export type SourceSyncCreateNamespaceResponse =
  SourceSyncApiResponse<SourceSyncNamespace>

export type SourceSyncListNamespacesResponse = SourceSyncApiResponse<
  SourceSyncNamespace[]
>

export type SourceSyncGetNamespaceResponse =
  SourceSyncApiResponse<SourceSyncNamespace>

export type SourceSyncUpdateNamespaceRequest = {
  fileStorageConfig?: SourceSyncFileStorageConfig
  vectorStorageConfig?: SourceSyncVectorStorageConfig
  embeddingModelConfig?: SourceSyncEmbeddingModelConfig
  webScraperConfig?: SourceSyncWebScraperConfig
  notionConfig?: SourceSyncNotionConfig
  googleDriveConfig?: SourceSyncGoogleDriveConfig
  dropboxConfig?: SourceSyncDropboxConfig
  onedriveConfig?: SourceSyncOnedriveConfig
  boxConfig?: SourceSyncBoxConfig
  sharepointConfig?: SourceSyncSharepointConfig
}

export type SourceSyncUpdateNamespaceResponse =
  SourceSyncApiResponse<SourceSyncNamespace>

export type SourceSyncDeleteNamespaceResponse = SourceSyncApiResponse<{
  identifier: string
}>

// ************************************************************************* //
// ************************** ⬆⬆ NAMESPACES ⬆⬆ ************************** //
// ************************************************************************* //

// ************************************************************************* //
// ************************** ⬇⬇ CONNECTION ⬇⬇ ************************** //
// ************************************************************************* //

export type SourceSyncOAuth2ConnectionConfig = {
  authType: SourceSyncConnectionAuthType.OAUTH2
  credentials: {
    accessToken: string
    accessTokenExpiresAt: number
    refreshToken: string | null
    refreshTokenExpiresAt: number | null
  } | null
}

export type SourceSyncConnectionConfig = SourceSyncOAuth2ConnectionConfig

export type SourceSyncConnection = {
  id: string
  name: string
  connector: SourceSyncConnector
  status: SourceSyncConnectionStatus
  statusReason: string | null
  config: SourceSyncConnectionConfig
  clientRedirectUrl: string | null
  tenantId: string | null
  namespace: SourceSyncNamespaceRef
  organization: SourceSyncFaunaRef
  createdAt: SourceSyncFaunaTime
  updatedAt: SourceSyncFaunaTime
}

export type SourceSyncCreateConnectionRequest = {
  namespaceId: string
  name: string
  connector: SourceSyncConnector
  clientRedirectUrl?: string
}

export type SourceSyncCreateConnectionResponse = SourceSyncApiResponse<{
  connection: SourceSyncFaunaRef
  authorizationUrl: string
}>

export type SourceSyncListConnectionsRequest = {
  connector?: SourceSyncConnector
}

export type SourceSyncListConnectionsResponse = SourceSyncApiResponse<{
  connections: SourceSyncConnection[]
}>

export type SourceSyncGetConnectionRequest = {
  connectionId: string
}

export type SourceSyncGetConnectionResponse = SourceSyncApiResponse<{
  connection: SourceSyncConnection
}>

export type SourceSyncUpdateConnectionRequest = {
  namespaceId: string
  name?: string
  clientRedirectUrl?: string
}

export type SourceSyncUpdateConnectionResponse = SourceSyncApiResponse<{
  connection: SourceSyncConnection
  authorizationUrl: string
}>

export type SourceSyncRevokeConnectionRequest = {
  namespaceId: string
}

export type SourceSyncRevokeConnectionResponse = SourceSyncApiResponse<{
  connection: SourceSyncConnection
}>

// ************************************************************************* //
// ************************** ⬆⬆ CONNECTION ⬆⬆ ************************** //
// ************************************************************************* //

// ************************************************************************* //
// ************************ ⬇⬇ DATA INGESTION ⬇⬇ ************************ //
// ************************************************************************* //

export type SourceSyncScrapeOptions = {
  includeSelectors?: string[]
  excludeSelectors?: string[]
}

export type SourceSyncChunkConfig = {
  chunkSize: number
  chunkOverlap: number
}

export type SourceSyncIngestTextRequest = {
  namespaceId: string
  ingestConfig: {
    source: SourceSyncIngestionSource.TEXT
    config: {
      name?: string
      text: string
      metadata?: Record<string, any>
    }
    chunkConfig?: SourceSyncChunkConfig
  }
}

export type SourceSyncIngestFileRequest = {
  namespaceId: string
  file: File
  metadata?: Record<string, any>
  chunkConfig?: SourceSyncChunkConfig
}

export type SourceSyncIngestUrlsRequest = {
  namespaceId: string
  ingestConfig: {
    source: SourceSyncIngestionSource.URLS_LIST
    config: {
      urls: string[]
      scrapeOptions?: SourceSyncScrapeOptions
      metadata?: Record<string, any>
    }
    chunkConfig?: SourceSyncChunkConfig
  }
}

export type SourceSyncIngestSitemapRequest = {
  namespaceId: string
  ingestConfig: {
    source: SourceSyncIngestionSource.SITEMAP
    config: {
      url: string
      maxLinks?: number
      includePaths?: string[]
      excludePaths?: string[]
      scrapeOptions?: SourceSyncScrapeOptions
      metadata?: Record<string, any>
    }
    chunkConfig?: SourceSyncChunkConfig
  }
}

export type SourceSyncIngestWebsiteRequest = {
  namespaceId: string
  ingestConfig: {
    source: SourceSyncIngestionSource.WEBSITE
    config: {
      url: string
      maxDepth?: number
      maxLinks?: number
      includePaths?: string[]
      excludePaths?: string[]
      scrapeOptions?: SourceSyncScrapeOptions
      metadata?: Record<string, any>
    }
    chunkConfig?: SourceSyncChunkConfig
  }
}

export type SourceSyncIngestConnectorRequest = {
  namespaceId: string
  ingestConfig: {
    source: SourceSyncIngestionSource
    config: {
      connectionId: string
      metadata?: Record<string, any>
    }
    chunkConfig?: SourceSyncChunkConfig
  }
}

export type SourceSyncIngestResponse = SourceSyncApiResponse<{
  ingestJobRunId: string
  documents: {
    id: string
    status: SourceSyncIngestionStatus
    error: string | null
  }[]
}>

export type SourceSyncIngestJobRun = {
  id: string
  status: SourceSyncIngestJobRunStatus
  createdAt: SourceSyncFaunaTime
  updatedAt: SourceSyncFaunaTime
} & (
  | {
      status:
        | SourceSyncIngestJobRunStatus.QUEUED
        | SourceSyncIngestJobRunStatus.PRE_PROCESSING
    }
  | {
      status:
        | SourceSyncIngestJobRunStatus.PROCESSING
        | SourceSyncIngestJobRunStatus.COMPLETED
      documents: {
        queued: {
          id: string
          status: SourceSyncIngestionStatus
          error: string | null
        }[]
        processing: {
          id: string
          status: SourceSyncIngestionStatus
          error: string | null
        }[]
        completed: {
          id: string
          status: SourceSyncIngestionStatus
          error: string | null
        }[]
        failed: {
          id: string
          status: SourceSyncIngestionStatus
          error: string | null
        }[]
        cancelled: {
          id: string
          status: SourceSyncIngestionStatus
          error: string | null
        }[]
      }
    }
)

export type SourceSyncGetIngestJobRunStatusRequest = {
  ingestJobRunId: string
}

export type SourceSyncGetIngestJobRunStatusResponse =
  SourceSyncApiResponse<SourceSyncIngestJobRun>

// ************************************************************************* //
// ************************ ⬆⬆ DATA INGESTION ⬆⬆ ************************ //
// ************************************************************************* //

// ************************************************************************* //
// *************************** ⬇⬇ DOCUMENT ⬇⬇ *************************** //
// ************************************************************************* //

export type SourceSyncDocumentFilterConfig = {
  documentIds?: string[]
  documentExternalIds?: string[]
  documentConnectionIds?: string[]
  documentTypes?: SourceSyncDocumentType[]
  documentIngestionSources?: SourceSyncIngestionSource[]
  documentIngestionStatuses?: SourceSyncIngestionStatus[]
  metadata?: Record<string, any>
}

export type SourceSyncDocumentIncludeConfig = {
  documents?: boolean
  rawFileUrl?: boolean
  parsedTextFileUrl?: boolean
  statsBySource?: boolean
  statsByStatus?: boolean
}

export type SourceSyncDocument = {
  id: string
  name: string | null
  externalId: string
  documentType: SourceSyncDocumentType
  ingestionSource: SourceSyncIngestionSource
  ingestionStatus: SourceSyncIngestionStatus
  ingestionError: string | null
  ingestJob: SourceSyncFaunaRef
  ingestJobRun: SourceSyncFaunaRef
  connection: SourceSyncFaunaRef
  documentProperties: {
    mimeType?: string
    fileSize?: number
    characterCount?: number
    tokenCount?: number
    embeddingCount?: number
  } | null
  embeddingConfig: {
    provider?: SourceSyncEmbeddingModelProvider
    model?:
      | SourceSyncOpenAIEmbeddingModel
      | SourceSyncCohereEmbeddingModel
      | SourceSyncJinaEmbeddingModel
    dimensions?: number
    chunkSize?: number
    chunkOverlap?: number
  } | null
  providers: {
    fileStorage?: SourceSyncFileStorageType
    vectorStorage?: SourceSyncVectorStorageProvider
    embeddingModel?: SourceSyncEmbeddingModelProvider
    webScraper?: SourceSyncWebScraperProvider
  } | null
  metadata: Record<string, string | string[]>
  rawFileUrl: string | null
  parsedTextFileUrl: string | null
  tenantId: string | null
  namespace: SourceSyncNamespaceRef
  organization: SourceSyncFaunaRef
  lastSyncedAt: SourceSyncFaunaTime | null
  createdAt: SourceSyncFaunaTime
  updatedAt: SourceSyncFaunaTime
}

export type SourceSyncGetDocumentsRequest = {
  namespaceId: string
  filterConfig: SourceSyncDocumentFilterConfig
  includeConfig?: SourceSyncDocumentIncludeConfig
  pagination?: SourceSyncPaginationRequest
}

export type SourceSyncGetDocumentsResponse = SourceSyncApiResponse<
  {
    statsBySource: { source: SourceSyncIngestionSource; totalCount: number }[]
    statsByStatus: { status: SourceSyncIngestionStatus; totalCount: number }[]
    documents: SourceSyncDocument[]
  } & SourceSyncPaginationResponse
>

export type SourceSyncUpdateDocumentsRequest = {
  namespaceId: string
  filterConfig: SourceSyncDocumentFilterConfig
  data: {
    metadata: Record<string, string>
    $metadata: {
      $set?: Record<string, string | string[]>
      $append?: Record<string, string[]>
      $remove?: Record<string, string[]>
    }
  }
}

export type SourceSyncUpdateDocumentsResponse = SourceSyncApiResponse<{
  itemsUpdated: number
  documents: SourceSyncDocument[]
}>

export type SourceSyncDeleteDocumentsRequest = {
  namespaceId: string
  filterConfig: SourceSyncDocumentFilterConfig
}

export type SourceSyncDeleteDocumentsResponse = SourceSyncApiResponse<{
  itemsDeleted: number
  documents: SourceSyncFaunaRef[]
}>

export type SourceSyncResyncDocumentsRequest = {
  namespaceId: string
  filterConfig: SourceSyncDocumentFilterConfig
}

export type SourceSyncResyncDocumentsResponse = SourceSyncApiResponse<{
  itemsQueued: number
  itemsSkipped: number
  documents: {
    id: string
    status: 'QUEUED_FOR_RESYNC' | 'NOT_ELIGIBLE_FOR_RESYNC'
    error: string | null
  }[]
}>

// ************************************************************************* //
// *************************** ⬆⬆ DOCUMENT ⬆⬆ *************************** //
// ************************************************************************* //

// ************************************************************************* //
// **************************** ⬇⬇ SEARCH ⬇⬇ **************************** //
// ************************************************************************* //

export type SourceSyncSearchFilter = {
  metadata?: Record<string, string | string[]>
}

export type SourceSyncHybridConfig = {
  semanticWeight: number
  keywordWeight: number
}

export type SourceSyncSearchResult = {
  content: string
  score: number
  metadata: {
    id: string
    source: string
    type: string
  }
  document: Pick<
    SourceSyncDocument,
    'id' | 'name' | 'externalId' | 'documentType'
  > | null
}

export type SourceSyncSemanticSearchRequest = {
  query: string
  namespaceId: string
  topK?: number
  scoreThreshold?: number
  filter?: SourceSyncSearchFilter
  searchType?: SourceSyncSearchType.SEMANTIC
}

export type SourceSyncHybridSearchRequest = {
  query: string
  namespaceId: string
  topK?: number
  scoreThreshold?: number
  filter?: SourceSyncSearchFilter
  hybridConfig: SourceSyncHybridConfig
  searchType?: SourceSyncSearchType.HYBRID
}

export type SourceSyncSearchResponse = SourceSyncApiResponse<{
  results: SourceSyncSearchResult[]
}>

// ************************************************************************* //
// **************************** ⬆⬆ SEARCH ⬆⬆ **************************** //
// ************************************************************************* //

// ************************************************************************* //
// **************************** ⬇⬇ UTILS ⬇⬇ **************************** //
// ************************************************************************* //

export function isSourceSyncDocumentFromWebSource(
  document: SourceSyncDocument,
): boolean {
  return [
    SourceSyncIngestionSource.URLS_LIST,
    SourceSyncIngestionSource.WEBSITE,
    SourceSyncIngestionSource.SITEMAP,
  ].includes(document.ingestionSource)
}

export function isSourceSyncDocumentPending(
  document: SourceSyncDocument,
): boolean {
  return [
    SourceSyncIngestionStatus.PROCESSING,
    SourceSyncIngestionStatus.QUEUED,
  ].includes(document.ingestionStatus)
}

export function isSourceSyncDocumentNotPending(
  document: SourceSyncDocument,
): boolean {
  return !isSourceSyncDocumentPending(document)
}

export function isSourceSyncDocumentFromLocalFileSource(
  document: SourceSyncDocument,
): boolean {
  return document.ingestionSource === SourceSyncIngestionSource.LOCAL_FILE
}

export function isSourceSyncDocumentFromTextSource(
  document: SourceSyncDocument,
): boolean {
  return document.ingestionSource === SourceSyncIngestionSource.TEXT
}

export function isSourceSyncDocumentFromExternalSource(
  document: SourceSyncDocument,
): boolean {
  return (
    !isSourceSyncDocumentFromTextSource(document) &&
    !isSourceSyncDocumentFromWebSource(document) &&
    !isSourceSyncDocumentFromLocalFileSource(document)
  )
}

export function getMemoryName({ brainId }: { brainId: string }): string {
  return `sourcesyncbrain-${brainId}`
}

// ************************************************************************* //
// **************************** ⬆⬆ UTILS ⬆⬆ **************************** //
// ************************************************************************* //