Module pachyderm_sdk.api.logs

Classes

class LogLevel (*args, **kwds)

LogLevel selects a log level. Pachyderm services only have DEBUG, INFO, and ERROR.

Expand source code

class LogLevel(betterproto.Enum):
    """
    LogLevel selects a log level. Pachyderm services only have DEBUG, INFO, and
    ERROR.
    """

    LOG_LEVEL_UNSET = 0
    """Use default log level filtering."""

    LOG_LEVEL_DEBUG = 1
    """Include DEBUG, INFO, and ERROR logs."""

    LOG_LEVEL_INFO = 2
    """Include INFO and ERROR logs."""

    LOG_LEVEL_ERROR = 3
    """Include only ERROR logs."""

Ancestors

• betterproto.Enum
• enum.IntEnum
• builtins.int
• enum.ReprEnum
• enum.Enum

Class variables

var LOG_LEVEL_UNSET

Use default log level filtering.

var LOG_LEVEL_DEBUG

Include DEBUG, INFO, and ERROR logs.

var LOG_LEVEL_INFO

Include INFO and ERROR logs.

var LOG_LEVEL_ERROR

Include only ERROR logs.

class LogQuery (user: UserLogQuery = None, admin: AdminLogQuery = None)

LogQuery names a source of logs.

Expand source code

@dataclass(eq=False, repr=False)
class LogQuery(betterproto.Message):
    """LogQuery names a source of logs."""

    user: "UserLogQuery" = betterproto.message_field(1, group="query_type")
    """A user query; covering day-to-day Pachyderm use cases."""

    admin: "AdminLogQuery" = betterproto.message_field(2, group="query_type")
    """
    A more broad log query that requires special permissions; useful for
    debugging Pachyderm itself.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var user : UserLogQuery

A user query; covering day-to-day Pachyderm use cases.

var admin : AdminLogQuery

A more broad log query that requires special permissions; useful for debugging Pachyderm itself.

class AdminLogQuery (logql: str = None, pod: str = None, pod_container: PodContainer = None, app: str = None, master: PipelineLogQuery = None, storage: PipelineLogQuery = None, user: UserLogQuery = None)

AdminLogQuery(logql: str = None, pod: str = None, pod_container: 'PodContainer' = None, app: str = None, master: 'PipelineLogQuery' = None, storage: 'PipelineLogQuery' = None, user: 'UserLogQuery' = None)

Expand source code

@dataclass(eq=False, repr=False)
class AdminLogQuery(betterproto.Message):
    logql: str = betterproto.string_field(1, group="admin_type")
    """Arbitrary LogQL query."""

    pod: str = betterproto.string_field(2, group="admin_type")
    """A pod's logs (all containers)."""

    pod_container: "PodContainer" = betterproto.message_field(3, group="admin_type")
    """One container."""

    app: str = betterproto.string_field(4, group="admin_type")
    """One "app" (logql -> {app=X})."""

    master: "PipelineLogQuery" = betterproto.message_field(5, group="admin_type")
    """All master worker lines from a pipeline."""

    storage: "PipelineLogQuery" = betterproto.message_field(6, group="admin_type")
    """All storage container lines from a pipeline."""

    user: "UserLogQuery" = betterproto.message_field(7, group="admin_type")
    """All worker lines from a pipeline/job."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var logql : str

Arbitrary LogQL query.

var pod : str

A pod's logs (all containers).

var pod_container : PodContainer

One container.

var app : str

One "app" (logql -> {app=X}).

var master : PipelineLogQuery

All master worker lines from a pipeline.

var storage : PipelineLogQuery

All storage container lines from a pipeline.

var user : UserLogQuery

All worker lines from a pipeline/job.

class PodContainer (pod: str = None, container: str = None)

PodContainer identifies a container running in a Pod.

Expand source code

@dataclass(eq=False, repr=False)
class PodContainer(betterproto.Message):
    """PodContainer identifies a container running in a Pod."""

    pod: str = betterproto.string_field(1)
    """The full name of the pod."""

    container: str = betterproto.string_field(2)
    """The name of the container."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var pod : str

The full name of the pod.

var container : str

The name of the container.

class UserLogQuery (project: str = None, pipeline: PipelineLogQuery = None, datum: str = None, job: str = None, pipeline_job: PipelineJobLogQuery = None, job_datum: JobDatumLogQuery = None, pipeline_datum: PipelineDatumLogQuery = None)

A UserLogQuery selects logs that Pachyderm users need to see during normal Pachyderm operations.

Expand source code

@dataclass(eq=False, repr=False)
class UserLogQuery(betterproto.Message):
    """
    A UserLogQuery selects logs that Pachyderm users need to see during normal
    Pachyderm operations.
    """

    project: str = betterproto.string_field(1, group="user_type")
    """All pipelines in the project."""

    pipeline: "PipelineLogQuery" = betterproto.message_field(2, group="user_type")
    """One pipeline in a project."""

    datum: str = betterproto.string_field(3, group="user_type")
    """One datum, by hex-encoded ID."""

    job: str = betterproto.string_field(4, group="user_type")
    """One job by hex-encoded ID, across pipelines and projects."""

    pipeline_job: "PipelineJobLogQuery" = betterproto.message_field(
        5, group="user_type"
    )
    """One job in one pipeline."""

    job_datum: "JobDatumLogQuery" = betterproto.message_field(6, group="user_type")
    """One datum in one job."""

    pipeline_datum: "PipelineDatumLogQuery" = betterproto.message_field(
        7, group="user_type"
    )
    """One datum in one pipeline."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var project : str

All pipelines in the project.

var pipeline : PipelineLogQuery

One pipeline in a project.

var datum : str

One datum, by hex-encoded ID.

var job : str

One job by hex-encoded ID, across pipelines and projects.

var pipeline_job : PipelineJobLogQuery

One job in one pipeline.

var job_datum : JobDatumLogQuery

One datum in one job.

var pipeline_datum : PipelineDatumLogQuery

One datum in one pipeline.

class PipelineLogQuery (project: str = None, pipeline: str = None)

PipelineLogQuery selects all logs from all runs of a named pipeline.

Expand source code

@dataclass(eq=False, repr=False)
class PipelineLogQuery(betterproto.Message):
    """PipelineLogQuery selects all logs from all runs of a named pipeline."""

    project: str = betterproto.string_field(1)
    """The project the pipeline is in."""

    pipeline: str = betterproto.string_field(2)
    """The name of the pipeline."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var project : str

The project the pipeline is in.

var pipeline : str

The name of the pipeline.

class PipelineJobLogQuery (pipeline: PipelineLogQuery = None, job: str = None)

PipelineJobLogQuery selects all logs from a job that a named pipeline performed.

Expand source code

@dataclass(eq=False, repr=False)
class PipelineJobLogQuery(betterproto.Message):
    """
    PipelineJobLogQuery selects all logs from a job that a named pipeline
    performed.
    """

    pipeline: "PipelineLogQuery" = betterproto.message_field(1)
    """The pipeline."""

    job: str = betterproto.string_field(2)
    """The hex-encoded ID of the job."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var pipeline : PipelineLogQuery

The pipeline.

var job : str

The hex-encoded ID of the job.

class JobDatumLogQuery (job: str = None, datum: str = None)

JobDatumLogQuery returns logs from the processing of one datum that was part of the job.

Expand source code

@dataclass(eq=False, repr=False)
class JobDatumLogQuery(betterproto.Message):
    """
    JobDatumLogQuery returns logs from the processing of one datum that was
    part of the job.
    """

    job: str = betterproto.string_field(1)
    """The hex-encoded ID of the job."""

    datum: str = betterproto.string_field(2)
    """The hex-encoded ID of the datum."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var job : str

The hex-encoded ID of the job.

var datum : str

The hex-encoded ID of the datum.

class PipelineDatumLogQuery (pipeline: PipelineLogQuery = None, datum: str = None)

PipelineDatumLogQuery returns logs from one datum in one pipeline.

Expand source code

@dataclass(eq=False, repr=False)
class PipelineDatumLogQuery(betterproto.Message):
    """PipelineDatumLogQuery returns logs from one datum in one pipeline."""

    pipeline: "PipelineLogQuery" = betterproto.message_field(1)
    """The pipeline."""

    datum: str = betterproto.string_field(2)
    """The hex-encoded ID of the datum."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var pipeline : PipelineLogQuery

The pipeline.

var datum : str

The hex-encoded ID of the datum.

class LogFilter (time_range: TimeRangeLogFilter = None, limit: int = None, regex: RegexLogFilter = None, level: LogLevel = None, user_logs_only: bool = None)

A LogFilter selects which log lines are returned.

Expand source code

@dataclass(eq=False, repr=False)
class LogFilter(betterproto.Message):
    """A LogFilter selects which log lines are returned."""

    time_range: "TimeRangeLogFilter" = betterproto.message_field(1)
    """If set, only return logs in the provided time range."""

    limit: int = betterproto.uint64_field(2)
    """If set, return at maximum this number of logs."""

    regex: "RegexLogFilter" = betterproto.message_field(3)
    """If set, only return logs that match this regular expression."""

    level: "LogLevel" = betterproto.enum_field(4)
    """
    If set, only return logs that are greater than or equal to this log level.
    (DEBUG returns DEBUG, INFO, ERROR, INFO returns INFO and ERROR, etc.).
    """

    user_logs_only: bool = betterproto.bool_field(5)
    """
    If true, only return logs from user code, excluding matched log messages
    only related to internal Pachyderm operations.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var time_range : TimeRangeLogFilter

If set, only return logs in the provided time range.

var limit : int

If set, return at maximum this number of logs.

var regex : RegexLogFilter

If set, only return logs that match this regular expression.

var level : LogLevel

If set, only return logs that are greater than or equal to this log level. (DEBUG returns DEBUG, INFO, ERROR, INFO returns INFO and ERROR, etc.).

var user_logs_only : bool

If true, only return logs from user code, excluding matched log messages only related to internal Pachyderm operations.

class TimeRangeLogFilter (from_: datetime.datetime = None, until: datetime.datetime = None, offset: int = None)

A TimeRangeLogFilter selects logs within a time range. Either or both timestamps can be null. If from is after until, logs will be returns in reverse order. (The first log you see will always be from the "from" time.)

Expand source code

@dataclass(eq=False, repr=False)
class TimeRangeLogFilter(betterproto.Message):
    """
    A TimeRangeLogFilter selects logs within a time range.  Either or both
    timestamps can be null. If from is after until, logs will be returns in
    reverse order.  (The first log you see will always be from the "from"
    time.)
    """

    from_: datetime = betterproto.message_field(1)
    """
    Where in time to start returning logs from; includes logs with this exact
    timestamp.  If null, starts at the beginning of time.
    """

    until: datetime = betterproto.message_field(2)
    """
    Where in time to stop returning logs from; includes logs with this exact
    timestamp.  If null, ends at the end of time.
    """

    offset: int = betterproto.uint64_field(3)
    """
    Offset from which to return results, in the case of multiple entries from
    the same nanosecond.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var from_ : datetime.datetime

Where in time to start returning logs from; includes logs with this exact timestamp. If null, starts at the beginning of time.

var until : datetime.datetime

Where in time to stop returning logs from; includes logs with this exact timestamp. If null, ends at the end of time.

var offset : int

Offset from which to return results, in the case of multiple entries from the same nanosecond.

class RegexLogFilter (pattern: str = None, negate: bool = None)

RegexLogFilter(pattern: str = None, negate: bool = None)

Expand source code

@dataclass(eq=False, repr=False)
class RegexLogFilter(betterproto.Message):
    pattern: str = betterproto.string_field(1)
    negate: bool = betterproto.bool_field(2)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var pattern : str

var negate : bool

class GetLogsRequest (query: LogQuery = None, filter: LogFilter = None, tail: bool = None, want_paging_hint: bool = None)

GetLogsRequest(query: 'LogQuery' = None, filter: 'LogFilter' = None, tail: bool = None, want_paging_hint: bool = None)

Expand source code

@dataclass(eq=False, repr=False)
class GetLogsRequest(betterproto.Message):
    query: "LogQuery" = betterproto.message_field(1)
    filter: "LogFilter" = betterproto.message_field(2)
    tail: bool = betterproto.bool_field(3)
    want_paging_hint: bool = betterproto.bool_field(4)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var query : LogQuery

var filter : LogFilter

var tail : bool

var want_paging_hint : bool

class GetLogsResponse (paging_hint: PagingHint = None, log: LogMessage = None)

GetLogsResponse(paging_hint: 'PagingHint' = None, log: 'LogMessage' = None)

Expand source code

@dataclass(eq=False, repr=False)
class GetLogsResponse(betterproto.Message):
    paging_hint: "PagingHint" = betterproto.message_field(1, group="response_type")
    log: "LogMessage" = betterproto.message_field(2, group="response_type")

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var paging_hint : PagingHint

var log : LogMessage

class PagingHint (older: GetLogsRequest = None, newer: GetLogsRequest = None)

PagingHint(older: 'GetLogsRequest' = None, newer: 'GetLogsRequest' = None)

Expand source code

@dataclass(eq=False, repr=False)
class PagingHint(betterproto.Message):
    older: "GetLogsRequest" = betterproto.message_field(1)
    newer: "GetLogsRequest" = betterproto.message_field(2)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var older : GetLogsRequest

var newer : GetLogsRequest

class VerbatimLogMessage (line: bytes = None, timestamp: datetime.datetime = None)

VerbatimLogMessage(line: bytes = None, timestamp: datetime.datetime = None)

Expand source code

@dataclass(eq=False, repr=False)
class VerbatimLogMessage(betterproto.Message):
    line: bytes = betterproto.bytes_field(1)
    timestamp: datetime = betterproto.message_field(2)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var line : bytes

var timestamp : datetime.datetime

class LogMessage (verbatim: VerbatimLogMessage = None, object: betterproto_lib_google_protobuf.Struct = None, native_timestamp: datetime.datetime = None, pps_log_message: _pps__.LogMessage = None)

LogMessage(verbatim: 'VerbatimLogMessage' = None, object: 'betterproto_lib_google_protobuf.Struct' = None, native_timestamp: datetime.datetime = None, pps_log_message: '_pps__.LogMessage' = None)

Expand source code

@dataclass(eq=False, repr=False)
class LogMessage(betterproto.Message):
    verbatim: "VerbatimLogMessage" = betterproto.message_field(1)
    """The verbatim line from Loki"""

    object: "betterproto_lib_google_protobuf.Struct" = betterproto.message_field(2)
    """A raw JSON parse of the entire line"""

    native_timestamp: datetime = betterproto.message_field(3)
    """If a parseable timestamp was found in `fields`"""

    pps_log_message: "_pps__.LogMessage" = betterproto.message_field(4)
    """For code that wants to filter on pipeline/job/etc"""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var verbatim : VerbatimLogMessage

The verbatim line from Loki

var object : betterproto.lib.google.protobuf.Struct

A raw JSON parse of the entire line

var native_timestamp : datetime.datetime

If a parseable timestamp was found in fields

var pps_log_message : LogMessage

For code that wants to filter on pipeline/job/etc

class ApiStub (channel: grpc.Channel)

Expand source code

class ApiStub:

    def __init__(self, channel: "grpc.Channel"):
        self.__rpc_get_logs = channel.unary_stream(
            "/logs.API/GetLogs",
            request_serializer=GetLogsRequest.SerializeToString,
            response_deserializer=GetLogsResponse.FromString,
        )

    def get_logs(
        self,
        *,
        query: "LogQuery" = None,
        filter: "LogFilter" = None,
        tail: bool = False,
        want_paging_hint: bool = False
    ) -> Iterator["GetLogsResponse"]:

        request = GetLogsRequest()
        if query is not None:
            request.query = query
        if filter is not None:
            request.filter = filter
        request.tail = tail
        request.want_paging_hint = want_paging_hint

        for response in self.__rpc_get_logs(request):
            yield response

Methods

def get_logs(self, *, query: LogQuery = None, filter: LogFilter = None, tail: bool = False, want_paging_hint: bool = False) ‑> Iterator[GetLogsResponse]

Expand source code

def get_logs(
    self,
    *,
    query: "LogQuery" = None,
    filter: "LogFilter" = None,
    tail: bool = False,
    want_paging_hint: bool = False
) -> Iterator["GetLogsResponse"]:

    request = GetLogsRequest()
    if query is not None:
        request.query = query
    if filter is not None:
        request.filter = filter
    request.tail = tail
    request.want_paging_hint = want_paging_hint

    for response in self.__rpc_get_logs(request):
        yield response