Home › Notes › Logging

Logging

Logging

docker info --format '{{.LoggingDriver}}'
docker ps -q |xargs -n 1 docker inspect -f '{{.HostConfig.LogConfig.Type}}'

# Search logs in vscode. case and regex
\.ERROR|\.CRIT
  • Loki
    • Look at Fluentd, Logstash or Fluentbit for capture
    • Pulling logs from Loki API
      # use lens port-forwarding

curl -G -s "http://localhost:56322/loki/api/v1/query_range" \
     --data-urlencode 'query={container="backend"} |= "authenticated"' \
     --data-urlencode 'start=2024-01-31T01:00:00.000Z' \
     --data-urlencode 'end=2024-03-01T02:00:00.000Z' \
     --data-urlencode 'limit=5000' \
     --data-urlencode 'direction=FORWARD' |tee ./auth-logs-2.json

cat auth-logs.json|jq '.data.result[].values|.' | awk -F'--' '{print $2}' |sort -u > list1
cat list* |sort -u | awk '{print $5}' > list-final

# util to find time 721hrs away
# offset.py
#!/usr/bin/env python
from datetime import datetime, timedelta
import sys
timestamp_str = sys.argv[1]
timestamp = datetime.strptime(timestamp_str, "%Y-%m-%dT%H:%M:%S.%fZ")
new_timestamp = timestamp + timedelta(hours=721)
new_timestamp_str = new_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
print(new_timestamp_str)
    • Grafana Loki Learning
      • my grafana stack
        GCLOUD_FM_URL="https://fleet-management-prod-014.grafana.net" \
GCLOUD_FM_POLL_FREQUENCY="60s" \
GCLOUD_FM_HOSTED_ID="1138878" \
ARCH="amd64" \
GCLOUD_RW_API_KEY="glc_eyJvIjoiMTMxODczNCIsIm4iOiJzdGFjay0xMTM4ODc4LWFsbG95LXRva2VuMSIsImsiOiJOYzR3SjhiODAxcWxCV0EyQWc0MHFxOTMiLCJtIjp7InIiOiJwcm9kLXVzLXdlc3QtMCJ9fQ==" \
/bin/sh -c \
"$(curl -fsSL https://storage.googleapis.com/cloud-onboarding/alloy/scripts/install-linux.sh)"
        # vi /etc/alloy/config.alloy
# sudo systemctl restart alloy.service

prometheus.exporter.postgres "integrations_postgres_exporter" {
    data_source_names = ["postgresql://localhost:5432/postgres"]
}

discovery.relabel "integrations_postgres_exporter" {
    targets = prometheus.exporter.postgres.integrations_postgres_exporter.targets
    
    rule {                
        target_label = "job"
        replacement  = "integrations/postgres_exporter"
    }
}

prometheus.relabel "integrations_postgres_exporter" {
	forward_to = [prometheus.remote_write.metrics_service.receiver]

  rule {
        target_label = "instance"
        replacement  = constants.hostname
  }

	rule {
    source_labels = ["__name__"]
    regex         = "pg_settings_max_connections|pg_settings_superuser_reserved_connections|pg_stat_activity_count|pg_stat_activity_max_tx_duration|pg_stat_bgwriter_buffers_alloc_total|pg_stat_bgwriter_buffers_backend_fsync_total|pg_stat_bgwriter_buffers_backend_total|pg_stat_bgwriter_buffers_checkpoint_total|pg_stat_bgwriter_buffers_clean_total|pg_stat_database_blks_hit|pg_stat_database_blks_read|pg_stat_database_conflicts|pg_stat_database_deadlocks|pg_stat_database_numbackends|pg_stat_database_tup_deleted|pg_stat_database_tup_fetched|pg_stat_database_tup_inserted|pg_stat_database_tup_returned|pg_stat_database_tup_updated|pg_stat_database_xact_commit|pg_stat_database_xact_rollback|pg_up|up"
    action        = "keep"
  }
}

prometheus.scrape "integrations_postgres_exporter" {
    targets    = discovery.relabel.integrations_postgres_exporter.output
    forward_to = [prometheus.relabel.integrations_postgres_exporter.receiver]
    job_name   = "integrations/postgres_exporter"
}

local.file_match "logs_integrations_postgres_exporter" {
    path_targets = [{
        __address__ = "localhost",
        __path__    = "/var/log/postgresql/postgres.log",
        instance    = constants.hostname,
        job         = "integrations/postgres_exporter",
    }]
}
loki.source.file "logs_integrations_postgres_exporter" {
    targets    = local.file_match.logs_integrations_postgres_exporter.targets
    forward_to = [loki.write.grafana_cloud_loki.receiver]
}