airbyte-protocol.yaml

---
"$schema": http://json-schema.org/draft-07/schema#
"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml
title: AirbyteProtocol
type: object
description: AirbyteProtocol structs
properties:
  airbyte_message:
    "$ref": "#/definitions/AirbyteMessage"
  configured_airbyte_catalog:
    "$ref": "#/definitions/ConfiguredAirbyteCatalog"
definitions:
  AirbyteMessage:
    type: object
    additionalProperties: true
    required:
      - type
    properties:
      type:
        description: "Message type"
        type: string
        enum:
          - RECORD
          - STATE
          - LOG
          - SPEC
          - CONNECTION_STATUS
          - CATALOG
      log:
        description: "log message: any kind of logging you want the platform to know about."
        "$ref": "#/definitions/AirbyteLogMessage"
      spec:
        "$ref": "#/definitions/ConnectorSpecification"
      # todo (cgardens) - prefer snake case for field names.
      connectionStatus:
        "$ref": "#/definitions/AirbyteConnectionStatus"
      catalog:
        description: "log message: any kind of logging you want the platform to know about."
        "$ref": "#/definitions/AirbyteCatalog"
      record:
        description: "record message: the record"
        "$ref": "#/definitions/AirbyteRecordMessage"
      state:
        description: "schema message: the state. Must be the last message produced. The platform uses this information"
        "$ref": "#/definitions/AirbyteStateMessage"
  AirbyteRecordMessage:
    type: object
    additionalProperties: true
    required:
      - stream
      - data
      - emitted_at
    properties:
      stream:
        description: "the name of this record's stream"
        type: string
      data:
        description: "the record data"
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      emitted_at:
        description: "when the data was emitted from the source. epoch in millisecond."
        type: integer
      namespace:
        description: "the namespace of this record's stream"
        type: string
  AirbyteStateMessage:
    type: object
    additionalProperties: true
    required:
      - data
    properties:
      data:
        description: "the state data"
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
  AirbyteLogMessage:
    type: object
    additionalProperties: true
    required:
      - level
      - message
    properties:
      level:
        description: "the type of logging"
        type: string
        enum:
          - FATAL
          - ERROR
          - WARN
          - INFO
          - DEBUG
          - TRACE
      message:
        description: "the log message"
        type: string
  AirbyteConnectionStatus:
    description: Airbyte connection status
    type: object
    additionalProperties: true
    required:
      - status
    properties:
      status:
        type: string
        enum:
          - SUCCEEDED
          - FAILED
      message:
        type: string
  AirbyteCatalog:
    description: Airbyte stream schema catalog
    type: object
    additionalProperties: true
    required:
      - streams
    properties:
      streams:
        type: array
        items:
          "$ref": "#/definitions/AirbyteStream"
  AirbyteStream:
    type: object
    additionalProperties: true
    required:
      - name
      - json_schema
      # todo (cgardens) - make required once sources are migrated
      # - supported_sync_modes
    properties:
      name:
        type: string
        description: Stream's name.
      json_schema:
        description: Stream schema using Json Schema specs.
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      supported_sync_modes:
        type: array
        items:
          "$ref": "#/definitions/SyncMode"
      source_defined_cursor:
        description: If the source defines the cursor field, then any other cursor field inputs will be ignored. If it does not, either the user_provided one is used, or the default one is used as a backup.
        type: boolean
      default_cursor_field:
        description: Path to the field that will be used to determine if a record is new or modified since the last sync. If not provided by the source, the end user will have to specify the comparable themselves.
        type: array
        items:
          type: string
      source_defined_primary_key:
        description: If the source defines the primary key, paths to the fields that will be used as a primary key. If not provided by the source, the end user will have to specify the primary key themselves.
        type: array
        items:
          type: array
          items:
            type: string
      namespace:
        description: Optional Source-defined namespace. Currently only used by JDBC destinations to determine what schema to write to. Airbyte streams from the same sources should have the same namespace.
        type: string
  ConfiguredAirbyteCatalog:
    description: Airbyte stream schema catalog
    type: object
    additionalProperties: true
    required:
      - streams
    properties:
      streams:
        type: array
        items:
          "$ref": "#/definitions/ConfiguredAirbyteStream"
  ConfiguredAirbyteStream:
    type: object
    additionalProperties: true
    required:
      - stream
      - sync_mode
      - destination_sync_mode
    properties:
      stream:
        "$ref": "#/definitions/AirbyteStream"
      sync_mode:
        "$ref": "#/definitions/SyncMode"
        default: full_refresh
      cursor_field:
        description: Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.
        type: array
        items:
          type: string
      destination_sync_mode:
        "$ref": "#/definitions/DestinationSyncMode"
        default: append
      primary_key:
        description: Paths to the fields that will be used as primary key. This field is REQUIRED if `destination_sync_mode` is `*_dedup`. Otherwise it is ignored.
        type: array
        items:
          type: array
          items:
            type: string
  SyncMode:
    type: string
    enum:
      - full_refresh
      - incremental
  DestinationSyncMode:
    type: string
    enum:
      - append
      - overwrite
      #- upsert_dedup # TODO chris: SCD Type 1 can be implemented later
      - append_dedup # SCD Type 1 & 2
  ConnectorSpecification:
    description: Specification of a connector (source/destination)
    type: object
    required:
      - connectionSpecification
    additionalProperties: true
    properties:
      # todo - split ConnectorSpecification into general vs connector type sections of properties
      # General Properties (Common to all connectors)
      # todo (cgardens) - prefer snake case for field names.
      documentationUrl:
        type: string
        format: uri
      # todo (cgardens) - prefer snake case for field names.
      changelogUrl:
        type: string
        format: uri
      # todo (cgardens) - prefer snake case for field names.
      connectionSpecification:
        description: ConnectorDefinition specific blob. Must be a valid JSON string.
        type: object
        existingJavaType: com.fasterxml.jackson.databind.JsonNode
      # Connector Type Properties (Common to all connectors from same type)
      # Source Connectors Properties
      supportsIncremental:
        description: If the connector supports incremental mode or not.
        type: boolean
      # Destination Connectors Properties
      supported_destination_sync_modes:
        description: List of destination sync modes supported by the connector
        type: array
        items:
          "$ref": "#/definitions/DestinationSyncMode"