Index

_

__init__() (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper method)

A

action (pyspark_pipeline_framework.core.audit.types.AuditEvent attribute) actor (pyspark_pipeline_framework.core.audit.types.AuditEvent attribute) AFTER_COMPONENT (pyspark_pipeline_framework.core.quality.types.CheckTiming attribute) after_component() (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks method) (pyspark_pipeline_framework.runner.checkpoint.CheckpointHooks method) (pyspark_pipeline_framework.runner.hooks.CompositeHooks method) (pyspark_pipeline_framework.runner.hooks.NoOpHooks method) (pyspark_pipeline_framework.runner.hooks.PipelineHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method) AFTER_PIPELINE (pyspark_pipeline_framework.core.quality.types.CheckTiming attribute) after_pipeline() (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks method) (pyspark_pipeline_framework.runner.checkpoint.CheckpointHooks method) (pyspark_pipeline_framework.runner.hooks.CompositeHooks method) (pyspark_pipeline_framework.runner.hooks.NoOpHooks method) (pyspark_pipeline_framework.runner.hooks.PipelineHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method)

AGGRESSIVE (pyspark_pipeline_framework.core.config.presets.ResiliencePolicies attribute) (pyspark_pipeline_framework.core.config.presets.RetryPolicies attribute) app_name (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) APPEND (pyspark_pipeline_framework.runtime.streaming.base.OutputMode attribute) ARRAY (pyspark_pipeline_framework.core.schema.definition.DataType attribute) audit (pyspark_pipeline_framework.core.config.hooks.HooksConfig attribute) audit_trail_path (pyspark_pipeline_framework.core.config.hooks.AuditConfig attribute) AuditAction (class in pyspark_pipeline_framework.core.audit.types) AuditConfig (class in pyspark_pipeline_framework.core.config.hooks) AuditEvent (class in pyspark_pipeline_framework.core.audit.types) AuditHooks (class in pyspark_pipeline_framework.runner.audit_hooks) AuditSink (class in pyspark_pipeline_framework.core.audit.sinks) AuditStatus (class in pyspark_pipeline_framework.core.audit.types) AVAILABLE_NOW (pyspark_pipeline_framework.runtime.streaming.base.TriggerType attribute) AVRO (pyspark_pipeline_framework.runtime.streaming.sinks.CloudFileFormat attribute) aws_region (pyspark_pipeline_framework.core.config.secrets.SecretsConfig attribute) AWS_SECRETS_MANAGER (pyspark_pipeline_framework.core.config.base.SecretsProvider attribute) AwsSecretsProvider (class in pyspark_pipeline_framework.core.secrets.providers)

B

backend (pyspark_pipeline_framework.core.config.hooks.MetricsConfig attribute) backoff_multiplier (pyspark_pipeline_framework.core.config.retry.RetryConfig attribute) BATCH (pyspark_pipeline_framework.core.config.base.PipelineMode attribute) before_component() (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks method) (pyspark_pipeline_framework.runner.checkpoint.CheckpointHooks method) (pyspark_pipeline_framework.runner.hooks.CompositeHooks method) (pyspark_pipeline_framework.runner.hooks.NoOpHooks method) (pyspark_pipeline_framework.runner.hooks.PipelineHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method) BEFORE_PIPELINE (pyspark_pipeline_framework.core.quality.types.CheckTiming attribute)

before_pipeline() (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks method) (pyspark_pipeline_framework.runner.checkpoint.CheckpointHooks method) (pyspark_pipeline_framework.runner.hooks.CompositeHooks method) (pyspark_pipeline_framework.runner.hooks.NoOpHooks method) (pyspark_pipeline_framework.runner.hooks.PipelineHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method) BINARY (pyspark_pipeline_framework.core.schema.definition.DataType attribute) BOOLEAN (pyspark_pipeline_framework.core.schema.definition.DataType attribute) bootstrap_servers (pyspark_pipeline_framework.runtime.streaming.sinks.KafkaStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sources.KafkaStreamingSource attribute)

C

cache_ttl_seconds (pyspark_pipeline_framework.core.config.secrets.SecretsConfig attribute) calculate_delay() (pyspark_pipeline_framework.core.resilience.retry.RetryExecutor method) call() (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker method) check_fn (pyspark_pipeline_framework.core.quality.types.DataQualityCheck attribute) check_name (pyspark_pipeline_framework.core.quality.types.CheckResult attribute) Checkpoint checkpoint_location (pyspark_pipeline_framework.runtime.streaming.base.StreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.ConsoleStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.DeltaStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.FileStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.ForeachBatchSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.IcebergStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.KafkaStreamingSink attribute) CheckpointHooks (class in pyspark_pipeline_framework.runner.checkpoint) CheckpointState (class in pyspark_pipeline_framework.runner.checkpoint) CheckpointStore (class in pyspark_pipeline_framework.runner.checkpoint) CheckResult (class in pyspark_pipeline_framework.core.quality.types) checks (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks property) CheckTiming (class in pyspark_pipeline_framework.core.quality.types) Circuit Breaker circuit_breaker (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) (pyspark_pipeline_framework.core.config.retry.ResiliencePolicy attribute) CIRCUIT_BREAKER_ONLY (pyspark_pipeline_framework.core.config.presets.ResiliencePolicies attribute) CircuitBreaker (class in pyspark_pipeline_framework.core.resilience.circuit_breaker) CircuitBreakerConfig (class in pyspark_pipeline_framework.core.config.retry) CircuitBreakerConfigs (class in pyspark_pipeline_framework.core.config.presets) CircuitBreakerOpenError CircuitState (class in pyspark_pipeline_framework.core.resilience.circuit_breaker) class_path (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) clear() (pyspark_pipeline_framework.core.secrets.resolver.SecretsCache method) CLIENT (pyspark_pipeline_framework.core.config.base.SparkDeployMode attribute) close() (pyspark_pipeline_framework.core.audit.sinks.AuditSink method) (pyspark_pipeline_framework.core.audit.sinks.CompositeAuditSink method) (pyspark_pipeline_framework.core.audit.sinks.FileAuditSink method) (pyspark_pipeline_framework.core.component.protocols.Resource method) CLOSED (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitState attribute) CloudFileFormat (class in pyspark_pipeline_framework.runtime.streaming.sinks) CloudStorageStreamingSink (class in pyspark_pipeline_framework.runtime.streaming.sinks) CLUSTER (pyspark_pipeline_framework.core.config.base.SparkDeployMode attribute) COMPLETE (pyspark_pipeline_framework.runtime.streaming.base.OutputMode attribute) completed_components (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute) (pyspark_pipeline_framework.runner.result.PipelineResult property) Component

COMPONENT_COMPLETED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) COMPONENT_CONFIG (pyspark_pipeline_framework.core.config.validator.ValidationPhase attribute) COMPONENT_FAILED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) component_name (pyspark_pipeline_framework.core.config.validator.ValidationError attribute) (pyspark_pipeline_framework.core.quality.types.DataQualityCheck attribute) (pyspark_pipeline_framework.runner.result.ComponentResult attribute) component_results (pyspark_pipeline_framework.runner.result.PipelineResult attribute) COMPONENT_RETRIED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) COMPONENT_STARTED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) component_type (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) ComponentConfig (class in pyspark_pipeline_framework.core.config.component) ComponentError ComponentExecutionError ComponentInstantiationError ComponentResult (class in pyspark_pipeline_framework.runner.result) components (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) ComponentType (class in pyspark_pipeline_framework.core.config.base) CompositeAuditSink (class in pyspark_pipeline_framework.core.audit.sinks) CompositeHooks (class in pyspark_pipeline_framework.runner.hooks) CompositeStreamingHooks (class in pyspark_pipeline_framework.runtime.streaming.hooks) compression (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink attribute) compute_pipeline_fingerprint() (in module pyspark_pipeline_framework.runner.checkpoint) config (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker property) (pyspark_pipeline_framework.core.resilience.retry.RetryExecutor property) CONFIG_LOADED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) CONFIG_SYNTAX (pyspark_pipeline_framework.core.config.validator.ValidationPhase attribute) ConfigFilter (class in pyspark_pipeline_framework.core.audit.filters) ConfigurableInstance (class in pyspark_pipeline_framework.core.component.protocols) connect_string (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) connection_string (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) CONSERVATIVE (pyspark_pipeline_framework.core.config.presets.ResiliencePolicies attribute) (pyspark_pipeline_framework.core.config.presets.RetryPolicies attribute) ConsoleStreamingSink (class in pyspark_pipeline_framework.runtime.streaming.sinks) consumer_group (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) CONTINUOUS (pyspark_pipeline_framework.runtime.streaming.base.TriggerType attribute) counter() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) (pyspark_pipeline_framework.core.metrics.registry.MeterRegistry method) created_at (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute) CRITICAL (pyspark_pipeline_framework.core.config.base.LogLevel attribute) CSV (pyspark_pipeline_framework.runtime.streaming.sinks.CloudFileFormat attribute) CUSTOM (pyspark_pipeline_framework.core.config.base.MetricsBackend attribute) custom_sql_check() (in module pyspark_pipeline_framework.core.quality.checks)

D

Data Quality Check data_type (pyspark_pipeline_framework.core.schema.definition.SchemaField attribute) DataFlow (class in pyspark_pipeline_framework.runtime.dataflow.base) DataQualityCheck (class in pyspark_pipeline_framework.core.quality.types) DataQualityError DataQualityHooks (class in pyspark_pipeline_framework.runner.quality_hooks) DataType (class in pyspark_pipeline_framework.core.schema.definition) DATE (pyspark_pipeline_framework.core.schema.definition.DataType attribute) DEBUG (pyspark_pipeline_framework.core.config.base.LogLevel attribute) DEFAULT (pyspark_pipeline_framework.core.config.presets.CircuitBreakerConfigs attribute) (pyspark_pipeline_framework.core.config.presets.ResiliencePolicies attribute) (pyspark_pipeline_framework.core.config.presets.RetryPolicies attribute) delete() (pyspark_pipeline_framework.runner.checkpoint.CheckpointStore method) (pyspark_pipeline_framework.runner.checkpoint.LocalCheckpointStore method)

DeltaStreamingSink (class in pyspark_pipeline_framework.runtime.streaming.sinks) DeltaStreamingSource (class in pyspark_pipeline_framework.runtime.streaming.sources) depends_on (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) deploy_mode (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) description (pyspark_pipeline_framework.core.schema.definition.SchemaDefinition attribute) details (pyspark_pipeline_framework.core.quality.types.CheckResult attribute) DEV (pyspark_pipeline_framework.core.config.base.Environment attribute) DOUBLE (pyspark_pipeline_framework.core.schema.definition.DataType attribute) driver_cores (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) driver_memory (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) dry_run() (in module pyspark_pipeline_framework.core.config.validator) (pyspark_pipeline_framework.runner.simple_runner.SimplePipelineRunner method) DryRunResult (class in pyspark_pipeline_framework.core.config.validator) duration_ms (pyspark_pipeline_framework.runner.result.ComponentResult attribute) dynamic_allocation (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute)

E

emit() (pyspark_pipeline_framework.core.audit.sinks.AuditSink method) (pyspark_pipeline_framework.core.audit.sinks.CompositeAuditSink method) (pyspark_pipeline_framework.core.audit.sinks.FileAuditSink method) (pyspark_pipeline_framework.core.audit.sinks.LoggingAuditSink method) emit_all() (pyspark_pipeline_framework.core.audit.sinks.AuditSink method) enabled (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) (pyspark_pipeline_framework.core.config.hooks.AuditConfig attribute) (pyspark_pipeline_framework.core.config.hooks.MetricsConfig attribute) END_OF_STREAM (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStartingPosition attribute) endpoint_url (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource attribute) ENV (pyspark_pipeline_framework.core.config.base.SecretsProvider attribute) Environment (class in pyspark_pipeline_framework.core.config.base) environment (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) EnvSecretsProvider (class in pyspark_pipeline_framework.core.secrets.providers) ERROR (pyspark_pipeline_framework.core.config.base.LogLevel attribute) (pyspark_pipeline_framework.core.schema.validator.ValidationSeverity attribute)

error (pyspark_pipeline_framework.core.secrets.base.SecretResolutionResult attribute) ERROR (pyspark_pipeline_framework.core.secrets.base.SecretResolutionStatus attribute) error (pyspark_pipeline_framework.runner.result.ComponentResult attribute) errors (pyspark_pipeline_framework.core.config.validator.DryRunResult attribute) (pyspark_pipeline_framework.core.config.validator.ValidationResult attribute) (pyspark_pipeline_framework.core.schema.validator.ValidationResult property) event_hub_name (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) EventHubsStartingPosition (class in pyspark_pipeline_framework.runtime.streaming.sources) EventHubsStreamingSource (class in pyspark_pipeline_framework.runtime.streaming.sources) execute() (pyspark_pipeline_framework.core.resilience.retry.RetryExecutor method) executor_cores (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) executor_memory (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) exists() (pyspark_pipeline_framework.runner.checkpoint.CheckpointStore method) (pyspark_pipeline_framework.runner.checkpoint.LocalCheckpointStore method) export_interval_seconds (pyspark_pipeline_framework.core.config.hooks.MetricsConfig attribute)

F

FAIL_ON_ERROR (pyspark_pipeline_framework.core.quality.types.FailureMode attribute) failed_component (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute) failed_components (pyspark_pipeline_framework.runner.result.PipelineResult property) FAILURE (pyspark_pipeline_framework.core.audit.types.AuditStatus attribute) (pyspark_pipeline_framework.runner.result.PipelineResultStatus attribute) failure_count (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker property) failure_mode (pyspark_pipeline_framework.core.quality.types.DataQualityCheck attribute) failure_threshold (pyspark_pipeline_framework.core.config.retry.CircuitBreakerConfig attribute) FailureMode (class in pyspark_pipeline_framework.core.quality.types) field_name (pyspark_pipeline_framework.core.schema.validator.ValidationIssue attribute) field_names() (pyspark_pipeline_framework.core.schema.definition.SchemaDefinition method) fields (pyspark_pipeline_framework.core.schema.definition.SchemaDefinition attribute) FILE (pyspark_pipeline_framework.core.config.base.SecretsProvider attribute) file_format (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.FileStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sources.FileStreamingSource attribute)

FileAuditSink (class in pyspark_pipeline_framework.core.audit.sinks) FileStreamingSink (class in pyspark_pipeline_framework.runtime.streaming.sinks) FileStreamingSource (class in pyspark_pipeline_framework.runtime.streaming.sources) FileToConsolePipeline (class in pyspark_pipeline_framework.examples.streaming) filter_condition (pyspark_pipeline_framework.examples.batch.ReadTableConfig attribute) FLOAT (pyspark_pipeline_framework.core.schema.definition.DataType attribute) ForeachBatchSink (class in pyspark_pipeline_framework.runtime.streaming.sinks) format (pyspark_pipeline_framework.core.config.hooks.LoggingConfig attribute) from_config() (pyspark_pipeline_framework.core.component.protocols.ConfigurableInstance class method) (pyspark_pipeline_framework.examples.batch.ReadCsv class method) (pyspark_pipeline_framework.examples.batch.ReadTable class method) (pyspark_pipeline_framework.examples.batch.SqlTransform class method) (pyspark_pipeline_framework.examples.batch.WriteCsv class method) (pyspark_pipeline_framework.examples.batch.WriteTable class method) from_file() (pyspark_pipeline_framework.runner.simple_runner.SimplePipelineRunner class method) from_struct_type() (in module pyspark_pipeline_framework.runtime.schema_converter)

G

gauge() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) (pyspark_pipeline_framework.core.metrics.registry.MeterRegistry method) get_component() (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig method) get_counter() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) get_execution_order() (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig method) get_field() (pyspark_pipeline_framework.core.schema.definition.SchemaDefinition method)

get_gauge() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) get_metrics() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) (pyspark_pipeline_framework.core.metrics.registry.MeterRegistry method) get_or_create() (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper class method) get_timer_count() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) get_timer_total() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method)

H

HALF_OPEN (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitState attribute) half_open_max_calls (pyspark_pipeline_framework.core.config.retry.CircuitBreakerConfig attribute) header (pyspark_pipeline_framework.examples.batch.ReadCsvConfig attribute) (pyspark_pipeline_framework.examples.batch.WriteCsvConfig attribute)

HOCON Hook hooks (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) HooksConfig (class in pyspark_pipeline_framework.core.config.hooks)

I

IcebergStreamingSink (class in pyspark_pipeline_framework.runtime.streaming.sinks) IcebergStreamingSource (class in pyspark_pipeline_framework.runtime.streaming.sources) include_data_samples (pyspark_pipeline_framework.core.config.hooks.AuditConfig attribute) infer_schema (pyspark_pipeline_framework.examples.batch.ReadCsvConfig attribute) INFO (pyspark_pipeline_framework.core.config.base.LogLevel attribute) initial_delay_seconds (pyspark_pipeline_framework.core.config.retry.RetryConfig attribute) InMemoryRegistry (class in pyspark_pipeline_framework.core.metrics.registry) input_schema (pyspark_pipeline_framework.core.component.protocols.SchemaContract property) (pyspark_pipeline_framework.runtime.dataflow.schema.SchemaAwareDataFlow property) input_view (pyspark_pipeline_framework.examples.batch.WriteCsvConfig attribute) (pyspark_pipeline_framework.examples.batch.WriteTableConfig attribute)

instantiate_component() (in module pyspark_pipeline_framework.runtime.loader) instantiated (pyspark_pipeline_framework.core.config.validator.DryRunResult attribute) INTEGER (pyspark_pipeline_framework.core.schema.definition.DataType attribute) interval (pyspark_pipeline_framework.runtime.streaming.base.TriggerConfig attribute) is_connect_mode (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper property) is_retryable() (pyspark_pipeline_framework.core.resilience.retry.RetryExecutor method) is_valid (pyspark_pipeline_framework.core.config.validator.DryRunResult property) (pyspark_pipeline_framework.core.config.validator.ValidationResult property) issues (pyspark_pipeline_framework.core.schema.validator.ValidationResult attribute)

J

JSON (pyspark_pipeline_framework.core.config.base.LogFormat attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.CloudFileFormat attribute)

K

KafkaStreamingSink (class in pyspark_pipeline_framework.runtime.streaming.sinks) KafkaStreamingSource (class in pyspark_pipeline_framework.runtime.streaming.sources) KafkaToDeltaPipeline (class in pyspark_pipeline_framework.examples.streaming)

key (pyspark_pipeline_framework.core.secrets.base.SecretsReference attribute) KinesisStartingPosition (class in pyspark_pipeline_framework.runtime.streaming.sources) KinesisStreamingSource (class in pyspark_pipeline_framework.runtime.streaming.sources)

L

LATEST (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStartingPosition attribute) level (pyspark_pipeline_framework.core.config.hooks.LoggingConfig attribute) list_available_components() (in module pyspark_pipeline_framework.runtime.loader) load() (pyspark_pipeline_framework.runner.checkpoint.CheckpointStore method) (pyspark_pipeline_framework.runner.checkpoint.LocalCheckpointStore method) load_checkpoint_for_resume() (in module pyspark_pipeline_framework.runner.checkpoint) load_component_class() (in module pyspark_pipeline_framework.runtime.loader) load_from_env() (in module pyspark_pipeline_framework.core.config.loader) load_from_file() (in module pyspark_pipeline_framework.core.config.loader) load_from_string() (in module pyspark_pipeline_framework.core.config.loader)

LocalCheckpointStore (class in pyspark_pipeline_framework.runner.checkpoint) LogFormat (class in pyspark_pipeline_framework.core.config.base) logger (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks property) (pyspark_pipeline_framework.runtime.dataflow.base.DataFlow property) logging (pyspark_pipeline_framework.core.config.hooks.HooksConfig attribute) LoggingAuditSink (class in pyspark_pipeline_framework.core.audit.sinks) LoggingConfig (class in pyspark_pipeline_framework.core.config.hooks) LoggingHooks (class in pyspark_pipeline_framework.runner.hooks_builtin) LoggingStreamingHooks (class in pyspark_pipeline_framework.runtime.streaming.hooks) LogLevel (class in pyspark_pipeline_framework.core.config.base) LONG (pyspark_pipeline_framework.core.schema.definition.DataType attribute)

M

MAP (pyspark_pipeline_framework.core.schema.definition.DataType attribute) master (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) max_attempts (pyspark_pipeline_framework.core.config.retry.RetryConfig attribute) max_delay_seconds (pyspark_pipeline_framework.core.config.retry.RetryConfig attribute) max_events_per_trigger (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) max_failures (pyspark_pipeline_framework.core.quality.types.DataQualityCheck attribute) max_fetch_records_per_shard (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource attribute) max_fetch_time_per_shard_sec (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource attribute) message (pyspark_pipeline_framework.core.config.validator.ValidationError attribute) (pyspark_pipeline_framework.core.quality.types.CheckResult attribute) (pyspark_pipeline_framework.core.schema.validator.ValidationIssue attribute) metadata (pyspark_pipeline_framework.core.audit.types.AuditEvent attribute) (pyspark_pipeline_framework.core.schema.definition.SchemaField attribute) MeterRegistry (class in pyspark_pipeline_framework.core.metrics.registry) metrics (pyspark_pipeline_framework.core.config.hooks.HooksConfig attribute) MetricsBackend (class in pyspark_pipeline_framework.core.config.base) MetricsConfig (class in pyspark_pipeline_framework.core.config.hooks) MetricsHooks (class in pyspark_pipeline_framework.runner.hooks_builtin) mode (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) (pyspark_pipeline_framework.examples.batch.WriteCsvConfig attribute) (pyspark_pipeline_framework.examples.batch.WriteTableConfig attribute) module pyspark_pipeline_framework.core.audit.filters pyspark_pipeline_framework.core.audit.sinks pyspark_pipeline_framework.core.audit.types pyspark_pipeline_framework.core.component.base pyspark_pipeline_framework.core.component.exceptions pyspark_pipeline_framework.core.component.protocols pyspark_pipeline_framework.core.config.base pyspark_pipeline_framework.core.config.component pyspark_pipeline_framework.core.config.hooks pyspark_pipeline_framework.core.config.loader pyspark_pipeline_framework.core.config.pipeline pyspark_pipeline_framework.core.config.presets pyspark_pipeline_framework.core.config.retry pyspark_pipeline_framework.core.config.secrets pyspark_pipeline_framework.core.config.spark pyspark_pipeline_framework.core.config.validator pyspark_pipeline_framework.core.metrics.registry pyspark_pipeline_framework.core.quality.checks pyspark_pipeline_framework.core.quality.types pyspark_pipeline_framework.core.resilience.circuit_breaker pyspark_pipeline_framework.core.resilience.retry pyspark_pipeline_framework.core.schema.definition pyspark_pipeline_framework.core.schema.validator pyspark_pipeline_framework.core.secrets.audit pyspark_pipeline_framework.core.secrets.base pyspark_pipeline_framework.core.secrets.providers pyspark_pipeline_framework.core.secrets.resolver pyspark_pipeline_framework.examples.batch pyspark_pipeline_framework.examples.streaming pyspark_pipeline_framework.runner.audit_hooks pyspark_pipeline_framework.runner.checkpoint pyspark_pipeline_framework.runner.hooks pyspark_pipeline_framework.runner.hooks_builtin pyspark_pipeline_framework.runner.quality_hooks pyspark_pipeline_framework.runner.result pyspark_pipeline_framework.runner.simple_runner pyspark_pipeline_framework.runtime.dataflow.base pyspark_pipeline_framework.runtime.dataflow.schema pyspark_pipeline_framework.runtime.loader pyspark_pipeline_framework.runtime.schema_converter pyspark_pipeline_framework.runtime.session.wrapper pyspark_pipeline_framework.runtime.streaming.base pyspark_pipeline_framework.runtime.streaming.hooks pyspark_pipeline_framework.runtime.streaming.sinks pyspark_pipeline_framework.runtime.streaming.sources

N

name (pyspark_pipeline_framework.core.component.base.PipelineComponent property) (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) (pyspark_pipeline_framework.core.quality.types.DataQualityCheck attribute) (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker property) (pyspark_pipeline_framework.core.schema.definition.SchemaField attribute) (pyspark_pipeline_framework.examples.batch.ReadCsv property) (pyspark_pipeline_framework.examples.batch.ReadTable property) (pyspark_pipeline_framework.examples.batch.SqlTransform property) (pyspark_pipeline_framework.examples.batch.WriteCsv property) (pyspark_pipeline_framework.examples.batch.WriteTable property) (pyspark_pipeline_framework.examples.streaming.FileToConsolePipeline property) (pyspark_pipeline_framework.examples.streaming.KafkaToDeltaPipeline property) (pyspark_pipeline_framework.runtime.dataflow.base.DataFlow property)

NO_RETRY (pyspark_pipeline_framework.core.config.presets.RetryPolicies attribute) NONE (pyspark_pipeline_framework.core.config.presets.ResiliencePolicies attribute) NoOpHooks (class in pyspark_pipeline_framework.runner.hooks) NoOpStreamingHooks (class in pyspark_pipeline_framework.runtime.streaming.hooks) NOT_FOUND (pyspark_pipeline_framework.core.secrets.base.SecretResolutionStatus attribute) null_check() (in module pyspark_pipeline_framework.core.quality.checks) nullable (pyspark_pipeline_framework.core.schema.definition.SchemaField attribute) num_executors (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) num_partitions (pyspark_pipeline_framework.runtime.streaming.sources.RateStreamingSource attribute)

O

on_batch_progress() (pyspark_pipeline_framework.runtime.streaming.hooks.CompositeStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.LoggingStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.NoOpStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.StreamingHooks method) on_circuit_breaker_state_change() (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks method) (pyspark_pipeline_framework.runner.checkpoint.CheckpointHooks method) (pyspark_pipeline_framework.runner.hooks.CompositeHooks method) (pyspark_pipeline_framework.runner.hooks.NoOpHooks method) (pyspark_pipeline_framework.runner.hooks.PipelineHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method) on_component_failure() (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks method) (pyspark_pipeline_framework.runner.checkpoint.CheckpointHooks method) (pyspark_pipeline_framework.runner.hooks.CompositeHooks method) (pyspark_pipeline_framework.runner.hooks.NoOpHooks method) (pyspark_pipeline_framework.runner.hooks.PipelineHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method) on_query_start() (pyspark_pipeline_framework.runtime.streaming.hooks.CompositeStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.LoggingStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.NoOpStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.StreamingHooks method) on_query_terminated() (pyspark_pipeline_framework.runtime.streaming.hooks.CompositeStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.LoggingStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.NoOpStreamingHooks method) (pyspark_pipeline_framework.runtime.streaming.hooks.StreamingHooks method) on_retry_attempt() (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks method) (pyspark_pipeline_framework.runner.checkpoint.CheckpointHooks method) (pyspark_pipeline_framework.runner.hooks.CompositeHooks method) (pyspark_pipeline_framework.runner.hooks.NoOpHooks method) (pyspark_pipeline_framework.runner.hooks.PipelineHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.LoggingHooks method) (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method)

ONCE (pyspark_pipeline_framework.runtime.streaming.base.TriggerType attribute) OPEN (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitState attribute) open() (pyspark_pipeline_framework.core.component.protocols.Resource method) OPENTELEMETRY (pyspark_pipeline_framework.core.config.base.MetricsBackend attribute) operation_timeout (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) options (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sources.DeltaStreamingSource attribute) (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) (pyspark_pipeline_framework.runtime.streaming.sources.FileStreamingSource attribute) (pyspark_pipeline_framework.runtime.streaming.sources.IcebergStreamingSource attribute) (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource attribute) ORC (pyspark_pipeline_framework.runtime.streaming.sinks.CloudFileFormat attribute) output (pyspark_pipeline_framework.core.config.hooks.LoggingConfig attribute) output_mode (pyspark_pipeline_framework.runtime.streaming.base.StreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.ConsoleStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.DeltaStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.FileStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.ForeachBatchSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.IcebergStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.KafkaStreamingSink attribute) output_schema (pyspark_pipeline_framework.core.component.protocols.SchemaContract property) (pyspark_pipeline_framework.runtime.dataflow.schema.SchemaAwareDataFlow property) output_table (pyspark_pipeline_framework.examples.batch.WriteTableConfig attribute) output_view (pyspark_pipeline_framework.examples.batch.ReadCsvConfig attribute) (pyspark_pipeline_framework.examples.batch.ReadTableConfig attribute) (pyspark_pipeline_framework.examples.batch.SqlTransformConfig attribute) OutputMode (class in pyspark_pipeline_framework.runtime.streaming.base)

P

PARQUET (pyspark_pipeline_framework.runtime.streaming.sinks.CloudFileFormat attribute) PARTIAL_SUCCESS (pyspark_pipeline_framework.runner.result.PipelineResultStatus attribute) partition_by (pyspark_pipeline_framework.examples.batch.WriteTableConfig attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.DeltaStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.FileStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.IcebergStreamingSink attribute) passed (pyspark_pipeline_framework.core.quality.types.CheckResult attribute) path (pyspark_pipeline_framework.examples.batch.ReadCsvConfig attribute) (pyspark_pipeline_framework.examples.batch.WriteCsvConfig attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.DeltaStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sinks.FileStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sources.DeltaStreamingSource attribute) (pyspark_pipeline_framework.runtime.streaming.sources.FileStreamingSource attribute) phase (pyspark_pipeline_framework.core.config.validator.ValidationError attribute) Pipeline Fingerprint PIPELINE_COMPLETED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) PIPELINE_FAILED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) pipeline_fingerprint (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute) pipeline_name (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute) (pyspark_pipeline_framework.runner.result.PipelineResult attribute) PIPELINE_STARTED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) PipelineComponent (class in pyspark_pipeline_framework.core.component.base) PipelineConfig (class in pyspark_pipeline_framework.core.config.pipeline) PipelineConfigChangedError PipelineHooks (class in pyspark_pipeline_framework.runner.hooks) PipelineMode (class in pyspark_pipeline_framework.core.config.base) PipelineResult (class in pyspark_pipeline_framework.runner.result) PipelineResultStatus (class in pyspark_pipeline_framework.runner.result) process_batch (pyspark_pipeline_framework.runtime.streaming.sinks.ForeachBatchSink attribute) PROCESSING_TIME (pyspark_pipeline_framework.runtime.streaming.base.TriggerType attribute) PROD (pyspark_pipeline_framework.core.config.base.Environment attribute) PROMETHEUS (pyspark_pipeline_framework.core.config.base.MetricsBackend attribute) provider (pyspark_pipeline_framework.core.config.secrets.SecretsConfig attribute) (pyspark_pipeline_framework.core.secrets.base.SecretsReference attribute) provider_name (pyspark_pipeline_framework.core.secrets.base.SecretsProvider property) (pyspark_pipeline_framework.core.secrets.providers.AwsSecretsProvider property) (pyspark_pipeline_framework.core.secrets.providers.EnvSecretsProvider property) (pyspark_pipeline_framework.core.secrets.providers.VaultSecretsProvider property) push_gateway_url (pyspark_pipeline_framework.core.config.hooks.MetricsConfig attribute) pyspark_pipeline_framework.core.audit.filters module pyspark_pipeline_framework.core.audit.sinks module pyspark_pipeline_framework.core.audit.types module pyspark_pipeline_framework.core.component.base module pyspark_pipeline_framework.core.component.exceptions module pyspark_pipeline_framework.core.component.protocols module pyspark_pipeline_framework.core.config.base module pyspark_pipeline_framework.core.config.component module pyspark_pipeline_framework.core.config.hooks module pyspark_pipeline_framework.core.config.loader module pyspark_pipeline_framework.core.config.pipeline module pyspark_pipeline_framework.core.config.presets module

pyspark_pipeline_framework.core.config.retry module pyspark_pipeline_framework.core.config.secrets module pyspark_pipeline_framework.core.config.spark module pyspark_pipeline_framework.core.config.validator module pyspark_pipeline_framework.core.metrics.registry module pyspark_pipeline_framework.core.quality.checks module pyspark_pipeline_framework.core.quality.types module pyspark_pipeline_framework.core.resilience.circuit_breaker module pyspark_pipeline_framework.core.resilience.retry module pyspark_pipeline_framework.core.schema.definition module pyspark_pipeline_framework.core.schema.validator module pyspark_pipeline_framework.core.secrets.audit module pyspark_pipeline_framework.core.secrets.base module pyspark_pipeline_framework.core.secrets.providers module pyspark_pipeline_framework.core.secrets.resolver module pyspark_pipeline_framework.examples.batch module pyspark_pipeline_framework.examples.streaming module pyspark_pipeline_framework.runner.audit_hooks module pyspark_pipeline_framework.runner.checkpoint module pyspark_pipeline_framework.runner.hooks module pyspark_pipeline_framework.runner.hooks_builtin module pyspark_pipeline_framework.runner.quality_hooks module pyspark_pipeline_framework.runner.result module pyspark_pipeline_framework.runner.simple_runner module pyspark_pipeline_framework.runtime.dataflow.base module pyspark_pipeline_framework.runtime.dataflow.schema module pyspark_pipeline_framework.runtime.loader module pyspark_pipeline_framework.runtime.schema_converter module pyspark_pipeline_framework.runtime.session.wrapper module pyspark_pipeline_framework.runtime.streaming.base module pyspark_pipeline_framework.runtime.streaming.hooks module pyspark_pipeline_framework.runtime.streaming.sinks module pyspark_pipeline_framework.runtime.streaming.sources module

Q

query_name (pyspark_pipeline_framework.runtime.streaming.base.StreamingSink property)

R

range_check() (in module pyspark_pipeline_framework.core.quality.checks) RateStreamingSource (class in pyspark_pipeline_framework.runtime.streaming.sources) read_stream() (pyspark_pipeline_framework.runtime.streaming.base.StreamingSource method) (pyspark_pipeline_framework.runtime.streaming.sources.DeltaStreamingSource method) (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource method) (pyspark_pipeline_framework.runtime.streaming.sources.FileStreamingSource method) (pyspark_pipeline_framework.runtime.streaming.sources.IcebergStreamingSource method) (pyspark_pipeline_framework.runtime.streaming.sources.KafkaStreamingSource method) (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource method) (pyspark_pipeline_framework.runtime.streaming.sources.RateStreamingSource method) ReadCsv (class in pyspark_pipeline_framework.examples.batch) ReadCsvConfig (class in pyspark_pipeline_framework.examples.batch) ReadTable (class in pyspark_pipeline_framework.examples.batch) ReadTableConfig (class in pyspark_pipeline_framework.examples.batch) receiver_timeout (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) record_failure() (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker method) record_success() (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker method) reference (pyspark_pipeline_framework.core.secrets.base.SecretResolutionResult attribute) region (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource attribute) register() (pyspark_pipeline_framework.core.secrets.resolver.SecretsResolver method) (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method) register_all() (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks method) registry (pyspark_pipeline_framework.runner.hooks_builtin.MetricsHooks property) REQUIRED_FIELDS (pyspark_pipeline_framework.core.config.validator.ValidationPhase attribute) reset() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker method) (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper class method) resilience (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) ResiliencePolicies (class in pyspark_pipeline_framework.core.config.presets) ResiliencePolicy (class in pyspark_pipeline_framework.core.config.retry) RESILIENT (pyspark_pipeline_framework.core.config.presets.CircuitBreakerConfigs attribute) resolve() (pyspark_pipeline_framework.core.secrets.audit.SecretsAuditLogger method) (pyspark_pipeline_framework.core.secrets.base.SecretsProvider method) (pyspark_pipeline_framework.core.secrets.providers.AwsSecretsProvider method) (pyspark_pipeline_framework.core.secrets.providers.EnvSecretsProvider method) (pyspark_pipeline_framework.core.secrets.providers.VaultSecretsProvider method) (pyspark_pipeline_framework.core.secrets.resolver.SecretsCache method) (pyspark_pipeline_framework.core.secrets.resolver.SecretsResolver method)

resolve_all() (pyspark_pipeline_framework.core.secrets.audit.SecretsAuditLogger method) (pyspark_pipeline_framework.core.secrets.base.SecretsProvider method) (pyspark_pipeline_framework.core.secrets.resolver.SecretsCache method) (pyspark_pipeline_framework.core.secrets.resolver.SecretsResolver method) Resource (class in pyspark_pipeline_framework.core.component.protocols) resource (pyspark_pipeline_framework.core.audit.types.AuditEvent attribute) results (pyspark_pipeline_framework.runner.quality_hooks.DataQualityHooks property) retention_days (pyspark_pipeline_framework.core.config.hooks.AuditConfig attribute) retries (pyspark_pipeline_framework.runner.result.ComponentResult attribute) RETRY (pyspark_pipeline_framework.core.audit.types.AuditStatus attribute) retry (pyspark_pipeline_framework.core.config.component.ComponentConfig attribute) (pyspark_pipeline_framework.core.config.retry.ResiliencePolicy attribute) retry_on_exceptions (pyspark_pipeline_framework.core.config.retry.RetryConfig attribute) RETRY_ONLY (pyspark_pipeline_framework.core.config.presets.ResiliencePolicies attribute) RetryConfig (class in pyspark_pipeline_framework.core.config.retry) RetryExecutor (class in pyspark_pipeline_framework.core.resilience.retry) RetryPolicies (class in pyspark_pipeline_framework.core.config.presets) row_count_check() (in module pyspark_pipeline_framework.core.quality.checks) rows_per_second (pyspark_pipeline_framework.runtime.streaming.sources.RateStreamingSource attribute) run() (pyspark_pipeline_framework.core.component.base.PipelineComponent method) (pyspark_pipeline_framework.examples.batch.ReadCsv method) (pyspark_pipeline_framework.examples.batch.ReadTable method) (pyspark_pipeline_framework.examples.batch.SqlTransform method) (pyspark_pipeline_framework.examples.batch.WriteCsv method) (pyspark_pipeline_framework.examples.batch.WriteTable method) (pyspark_pipeline_framework.runner.simple_runner.SimplePipelineRunner method) (pyspark_pipeline_framework.runtime.dataflow.base.DataFlow method) (pyspark_pipeline_framework.runtime.streaming.base.StreamingPipeline method) run_id (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute)

S

save() (pyspark_pipeline_framework.runner.checkpoint.CheckpointStore method) (pyspark_pipeline_framework.runner.checkpoint.LocalCheckpointStore method) schema (pyspark_pipeline_framework.runtime.streaming.sources.FileStreamingSource attribute) schema_check() (in module pyspark_pipeline_framework.core.quality.checks) SchemaAwareDataFlow (class in pyspark_pipeline_framework.runtime.dataflow.schema) SchemaContract (class in pyspark_pipeline_framework.core.component.protocols) SchemaDefinition (class in pyspark_pipeline_framework.core.schema.definition) SchemaField (class in pyspark_pipeline_framework.core.schema.definition) SchemaValidator (class in pyspark_pipeline_framework.core.schema.validator) scrub() (pyspark_pipeline_framework.core.audit.filters.ConfigFilter class method) SECRET_ACCESSED (pyspark_pipeline_framework.core.audit.types.AuditAction attribute) secret_prefix (pyspark_pipeline_framework.core.config.secrets.SecretsConfig attribute) SecretResolutionResult (class in pyspark_pipeline_framework.core.secrets.base) SecretResolutionStatus (class in pyspark_pipeline_framework.core.secrets.base) secrets (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) SecretsAuditLogger (class in pyspark_pipeline_framework.core.secrets.audit) SecretsCache (class in pyspark_pipeline_framework.core.secrets.resolver) SecretsConfig (class in pyspark_pipeline_framework.core.config.secrets) SecretsProvider (class in pyspark_pipeline_framework.core.config.base) (class in pyspark_pipeline_framework.core.secrets.base) SecretsReference (class in pyspark_pipeline_framework.core.secrets.base) SecretsResolver (class in pyspark_pipeline_framework.core.secrets.resolver) SENSITIVE (pyspark_pipeline_framework.core.config.presets.CircuitBreakerConfigs attribute) set_spark_session() (pyspark_pipeline_framework.runtime.dataflow.base.DataFlow method) (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper method) severity (pyspark_pipeline_framework.core.schema.validator.ValidationIssue attribute) SimplePipelineRunner (class in pyspark_pipeline_framework.runner.simple_runner) SINK (pyspark_pipeline_framework.core.config.base.ComponentType attribute) sink (pyspark_pipeline_framework.examples.streaming.FileToConsolePipeline property) (pyspark_pipeline_framework.examples.streaming.KafkaToDeltaPipeline property) (pyspark_pipeline_framework.runtime.streaming.base.StreamingPipeline property) SOURCE (pyspark_pipeline_framework.core.config.base.ComponentType attribute) source (pyspark_pipeline_framework.examples.streaming.FileToConsolePipeline property) (pyspark_pipeline_framework.examples.streaming.KafkaToDeltaPipeline property) (pyspark_pipeline_framework.runtime.streaming.base.StreamingPipeline property) source_component (pyspark_pipeline_framework.core.schema.validator.ValidationIssue attribute) (pyspark_pipeline_framework.core.schema.validator.ValidationResult attribute)

spark (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) (pyspark_pipeline_framework.runtime.dataflow.base.DataFlow property) (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper property) spark_conf (pyspark_pipeline_framework.core.config.spark.SparkConfig attribute) spark_context (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper property) SparkConfig (class in pyspark_pipeline_framework.core.config.spark) SparkDeployMode (class in pyspark_pipeline_framework.core.config.base) SparkSessionWrapper (class in pyspark_pipeline_framework.runtime.session.wrapper) sql (pyspark_pipeline_framework.examples.batch.SqlTransformConfig attribute) sql_context (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper property) SqlTransform (class in pyspark_pipeline_framework.examples.batch) SqlTransformConfig (class in pyspark_pipeline_framework.examples.batch) STAGING (pyspark_pipeline_framework.core.config.base.Environment attribute) START_OF_STREAM (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStartingPosition attribute) start_stream() (pyspark_pipeline_framework.runtime.streaming.base.StreamingPipeline method) starting_offsets (pyspark_pipeline_framework.runtime.streaming.sources.KafkaStreamingSource attribute) starting_position (pyspark_pipeline_framework.runtime.streaming.sources.EventHubsStreamingSource attribute) (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource attribute) state (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker property) status (pyspark_pipeline_framework.core.audit.types.AuditEvent attribute) (pyspark_pipeline_framework.core.secrets.base.SecretResolutionResult attribute) (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute) (pyspark_pipeline_framework.runner.result.PipelineResult attribute) stop() (pyspark_pipeline_framework.runtime.session.wrapper.SparkSessionWrapper method) stream_name (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStreamingSource attribute) STREAMING (pyspark_pipeline_framework.core.config.base.PipelineMode attribute) StreamingHooks (class in pyspark_pipeline_framework.runtime.streaming.hooks) StreamingPipeline (class in pyspark_pipeline_framework.runtime.streaming.base) StreamingSink (class in pyspark_pipeline_framework.runtime.streaming.base) StreamingSource (class in pyspark_pipeline_framework.runtime.streaming.base) STRING (pyspark_pipeline_framework.core.schema.definition.DataType attribute) STRUCT (pyspark_pipeline_framework.core.schema.definition.DataType attribute) structured (pyspark_pipeline_framework.core.config.hooks.LoggingConfig attribute) SUCCESS (pyspark_pipeline_framework.core.audit.types.AuditStatus attribute) (pyspark_pipeline_framework.core.secrets.base.SecretResolutionStatus attribute) success (pyspark_pipeline_framework.runner.result.ComponentResult attribute) SUCCESS (pyspark_pipeline_framework.runner.result.PipelineResultStatus attribute) success_count (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker property) success_threshold (pyspark_pipeline_framework.core.config.retry.CircuitBreakerConfig attribute)

T

table (pyspark_pipeline_framework.runtime.streaming.sinks.IcebergStreamingSink attribute) (pyspark_pipeline_framework.runtime.streaming.sources.IcebergStreamingSource attribute) table_name (pyspark_pipeline_framework.examples.batch.ReadTableConfig attribute) tags (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute) target_component (pyspark_pipeline_framework.core.schema.validator.ValidationIssue attribute) (pyspark_pipeline_framework.core.schema.validator.ValidationResult attribute) TEST (pyspark_pipeline_framework.core.config.base.Environment attribute) TEXT (pyspark_pipeline_framework.core.config.base.LogFormat attribute) THRESHOLD (pyspark_pipeline_framework.core.quality.types.FailureMode attribute) time_until_reset (pyspark_pipeline_framework.core.resilience.circuit_breaker.CircuitBreaker property) timeout_seconds (pyspark_pipeline_framework.core.config.retry.CircuitBreakerConfig attribute) timer() (pyspark_pipeline_framework.core.metrics.registry.InMemoryRegistry method) (pyspark_pipeline_framework.core.metrics.registry.MeterRegistry method) timestamp (pyspark_pipeline_framework.core.audit.types.AuditEvent attribute) TIMESTAMP (pyspark_pipeline_framework.core.schema.definition.DataType attribute) timing (pyspark_pipeline_framework.core.quality.types.DataQualityCheck attribute) to_dict() (pyspark_pipeline_framework.core.audit.types.AuditEvent method) to_spark_conf_dict() (pyspark_pipeline_framework.core.config.spark.SparkConfig method)

to_struct_type() (in module pyspark_pipeline_framework.runtime.schema_converter) topic (pyspark_pipeline_framework.runtime.streaming.sinks.KafkaStreamingSink attribute) topics (pyspark_pipeline_framework.runtime.streaming.sources.KafkaStreamingSource attribute) total_duration_ms (pyspark_pipeline_framework.runner.result.PipelineResult attribute) trace_id (pyspark_pipeline_framework.core.audit.types.AuditEvent attribute) (pyspark_pipeline_framework.runner.audit_hooks.AuditHooks property) transform() (pyspark_pipeline_framework.examples.streaming.FileToConsolePipeline method) (pyspark_pipeline_framework.runtime.streaming.base.StreamingPipeline method) TRANSFORMATION (pyspark_pipeline_framework.core.config.base.ComponentType attribute) trigger (pyspark_pipeline_framework.examples.streaming.FileToConsolePipeline property) (pyspark_pipeline_framework.examples.streaming.KafkaToDeltaPipeline property) (pyspark_pipeline_framework.runtime.streaming.base.StreamingPipeline property) trigger_type (pyspark_pipeline_framework.runtime.streaming.base.TriggerConfig attribute) TriggerConfig (class in pyspark_pipeline_framework.runtime.streaming.base) TriggerType (class in pyspark_pipeline_framework.runtime.streaming.base) TRIM_HORIZON (pyspark_pipeline_framework.runtime.streaming.sources.KinesisStartingPosition attribute) truncate (pyspark_pipeline_framework.runtime.streaming.sinks.ConsoleStreamingSink attribute) TYPE_RESOLUTION (pyspark_pipeline_framework.core.config.validator.ValidationPhase attribute)

U

unique_check() (in module pyspark_pipeline_framework.core.quality.checks)

UPDATE (pyspark_pipeline_framework.runtime.streaming.base.OutputMode attribute) updated_at (pyspark_pipeline_framework.runner.checkpoint.CheckpointState attribute)

V

valid (pyspark_pipeline_framework.core.schema.validator.ValidationResult attribute) validate() (pyspark_pipeline_framework.core.schema.validator.SchemaValidator method) validate_component_class() (in module pyspark_pipeline_framework.runtime.loader) validate_pipeline() (in module pyspark_pipeline_framework.core.config.validator) ValidationError (class in pyspark_pipeline_framework.core.config.validator) ValidationIssue (class in pyspark_pipeline_framework.core.schema.validator) ValidationPhase (class in pyspark_pipeline_framework.core.config.validator) ValidationResult (class in pyspark_pipeline_framework.core.config.validator) (class in pyspark_pipeline_framework.core.schema.validator)

ValidationSeverity (class in pyspark_pipeline_framework.core.schema.validator) value (pyspark_pipeline_framework.core.secrets.base.SecretResolutionResult attribute) VAULT (pyspark_pipeline_framework.core.config.base.SecretsProvider attribute) vault_namespace (pyspark_pipeline_framework.core.config.secrets.SecretsConfig attribute) vault_token (pyspark_pipeline_framework.core.config.secrets.SecretsConfig attribute) vault_url (pyspark_pipeline_framework.core.config.secrets.SecretsConfig attribute) VaultSecretsProvider (class in pyspark_pipeline_framework.core.secrets.providers) version (pyspark_pipeline_framework.core.config.pipeline.PipelineConfig attribute)

W

WARN_ONLY (pyspark_pipeline_framework.core.quality.types.FailureMode attribute) WARNING (pyspark_pipeline_framework.core.audit.types.AuditStatus attribute) (pyspark_pipeline_framework.core.config.base.LogLevel attribute) (pyspark_pipeline_framework.core.schema.validator.ValidationSeverity attribute) warnings (pyspark_pipeline_framework.core.config.validator.ValidationResult attribute) (pyspark_pipeline_framework.core.schema.validator.ValidationResult property) watermark_column (pyspark_pipeline_framework.runtime.streaming.base.StreamingSource property) watermark_delay (pyspark_pipeline_framework.runtime.streaming.base.StreamingSource property) with_retry() (in module pyspark_pipeline_framework.core.resilience.retry) write_stream() (pyspark_pipeline_framework.runtime.streaming.base.StreamingSink method) (pyspark_pipeline_framework.runtime.streaming.sinks.CloudStorageStreamingSink method) (pyspark_pipeline_framework.runtime.streaming.sinks.ConsoleStreamingSink method) (pyspark_pipeline_framework.runtime.streaming.sinks.DeltaStreamingSink method) (pyspark_pipeline_framework.runtime.streaming.sinks.FileStreamingSink method) (pyspark_pipeline_framework.runtime.streaming.sinks.ForeachBatchSink method) (pyspark_pipeline_framework.runtime.streaming.sinks.IcebergStreamingSink method) (pyspark_pipeline_framework.runtime.streaming.sinks.KafkaStreamingSink method)

WriteCsv (class in pyspark_pipeline_framework.examples.batch) WriteCsvConfig (class in pyspark_pipeline_framework.examples.batch) WriteTable (class in pyspark_pipeline_framework.examples.batch) WriteTableConfig (class in pyspark_pipeline_framework.examples.batch)