IBM Cloud Site Reliability Engineer (C1000-179) - Monitoring & Observability¶

// custom-metrics.js
const StatsD = require('node-statsd');

const statsd = new StatsD({
  host: 'localhost',
  port: 8125,
  prefix: 'myapp.',
  global_tags: ['env:production', 'service:api']
});

// Business metrics
function recordOrder(orderId, amount) {
  statsd.increment('orders.total');
  statsd.histogram('orders.value', amount);
  statsd.gauge('orders.in_progress', getInProgressOrders());
}

// Application metrics
function recordAPICall(endpoint, duration, statusCode) {
  statsd.timing(`api.${endpoint}.response_time`, duration);
  statsd.increment(`api.${endpoint}.requests`, 1, [`status:${statusCode}`]);

  if (statusCode >= 500) {
    statsd.increment('api.errors.5xx');
  } else if (statusCode >= 400) {
    statsd.increment('api.errors.4xx');
  }
}

// Database metrics
function recordDatabaseQuery(queryType, duration) {
  statsd.timing(`database.query.${queryType}`, duration);
  statsd.increment(`database.queries.${queryType}`);
}

// Cache metrics
function recordCacheOperation(operation, hit) {
  statsd.increment(`cache.${operation}.total`);
  if (hit) {
    statsd.increment(`cache.${operation}.hits`);
  } else {
    statsd.increment(`cache.${operation}.misses`);
  }
}

// Middleware for Express
function metricsMiddleware(req, res, next) {
  const start = Date.now();

  res.on('finish', () => {
    const duration = Date.now() - start;
    const route = req.route ? req.route.path : 'unknown';
    recordAPICall(route, duration, res.statusCode);
  });

  next();
}

module.exports = {
  recordOrder,
  recordAPICall,
  recordDatabaseQuery,
  recordCacheOperation,
  metricsMiddleware
};

// logger.js
const winston = require('winston');
const { LogDNA } = require('logdna-winston');

const logger = winston.createLogger({
  level: 'info',
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.errors({ stack: true }),
    winston.format.json()
  ),
  defaultMeta: {
    service: 'api-service',
    environment: process.env.NODE_ENV,
    version: process.env.APP_VERSION,
    hostname: process.env.HOSTNAME
  },
  transports: [
    new winston.transports.Console({
      format: winston.format.combine(
        winston.format.colorize(),
        winston.format.simple()
      )
    }),
    new LogDNA({
      key: process.env.LOGDNA_KEY,
      hostname: process.env.HOSTNAME,
      app: 'api-service',
      env: process.env.NODE_ENV,
      index_meta: true
    })
  ]
});

// Request logging middleware
function requestLogger(req, res, next) {
  const start = Date.now();

  // Log request
  logger.info('HTTP Request', {
    method: req.method,
    url: req.url,
    headers: {
      'user-agent': req.get('user-agent'),
      'x-correlation-id': req.get('x-correlation-id')
    },
    query: req.query,
    ip: req.ip
  });

  // Log response
  res.on('finish', () => {
    const duration = Date.now() - start;

    const logLevel = res.statusCode >= 500 ? 'error' : 
                     res.statusCode >= 400 ? 'warn' : 'info';

    logger.log(logLevel, 'HTTP Response', {
      method: req.method,
      url: req.url,
      status: res.statusCode,
      duration,
      correlationId: req.get('x-correlation-id')
    });
  });

  next();
}

// Error logging
function logError(error, context = {}) {
  logger.error('Application Error', {
    error: {
      message: error.message,
      stack: error.stack,
      name: error.name
    },
    ...context
  });
}

// Business event logging
function logBusinessEvent(eventType, data) {
  logger.info('Business Event', {
    eventType,
    data,
    timestamp: new Date().toISOString()
  });
}

module.exports = { logger, requestLogger, logError, logBusinessEvent };

// tracing.js
const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base');
const { registerInstrumentations } = require('@opentelemetry/instrumentation');
const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http');
const { ExpressInstrumentation } = require('@opentelemetry/instrumentation-express');
const { PgInstrumentation } = require('@opentelemetry/instrumentation-pg');
const { RedisInstrumentation } = require('@opentelemetry/instrumentation-redis');

const provider = new NodeTracerProvider({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'api-service',
    [SemanticResourceAttributes.SERVICE_VERSION]: process.env.APP_VERSION,
    [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: process.env.NODE_ENV
  })
});

// Jaeger exporter
const jaegerExporter = new JaegerExporter({
  endpoint: process.env.JAEGER_ENDPOINT || 'http://jaeger:14268/api/traces'
});

provider.addSpanProcessor(new BatchSpanProcessor(jaegerExporter));
provider.register();

// Register instrumentations
registerInstrumentations({
  instrumentations: [
    new HttpInstrumentation({
      requestHook: (span, request) => {
        span.setAttribute('http.user_agent', request.headers['user-agent']);
      }
    }),
    new ExpressInstrumentation(),
    new PgInstrumentation(),
    new RedisInstrumentation()
  ]
});

// Get tracer
const tracer = provider.getTracer('api-service');

module.exports = { tracer };

// business-logic.js
const { tracer } = require('./tracing');
const opentelemetry = require('@opentelemetry/api');

async function processOrder(orderId) {
  const span = tracer.startSpan('processOrder');
  span.setAttribute('order.id', orderId);
  span.setAttribute('order.type', 'standard');

  try {
    const context = opentelemetry.trace.setSpan(
      opentelemetry.context.active(),
      span
    );

    await opentelemetry.context.with(context, async () => {
      // Validate order
      await validateOrder(orderId);

      // Process payment
      await processPayment(orderId);

      // Update inventory
      await updateInventory(orderId);

      // Send notification
      await sendNotification(orderId);
    });

    span.setStatus({ code: opentelemetry.SpanStatusCode.OK });
    return { success: true };

  } catch (error) {
    span.setStatus({
      code: opentelemetry.SpanStatusCode.ERROR,
      message: error.message
    });
    span.recordException(error);
    throw error;
  } finally {
    span.end();
  }
}

async function validateOrder(orderId) {
  const span = tracer.startSpan('validateOrder', {
    parent: opentelemetry.trace.getActiveSpan()
  });

  try {
    // Validation logic
    const order = await db.getOrder(orderId);

    if (!order) {
      throw new Error('Order not found');
    }

    if (order.total < 0) {
      throw new Error('Invalid order total');
    }

    span.addEvent('Order validated', {
      'order.total': order.total,
      'order.items': order.items.length
    });

  } finally {
    span.end();
  }
}

// api-client.js
const axios = require('axios');
const opentelemetry = require('@opentelemetry/api');

async function callDownstreamService(url, data) {
  const span = tracer.startSpan('callDownstreamService');
  span.setAttribute('http.url', url);

  try {
    // Get current context
    const context = opentelemetry.trace.setSpan(
      opentelemetry.context.active(),
      span
    );

    // Inject trace context into headers
    const headers = {};
    opentelemetry.propagation.inject(context, headers);

    const response = await axios.post(url, data, { headers });

    span.setAttribute('http.status_code', response.status);
    return response.data;

  } catch (error) {
    span.recordException(error);
    throw error;
  } finally {
    span.end();
  }
}

// pagerduty.js
const axios = require('axios');

class PagerDutyClient {
  constructor(integrationKey) {
    this.integrationKey = integrationKey;
    this.apiUrl = 'https://events.pagerduty.com/v2/enqueue';
  }

  async triggerIncident(summary, details, severity = 'critical') {
    const payload = {
      routing_key: this.integrationKey,
      event_action: 'trigger',
      payload: {
        summary,
        severity,
        source: 'monitoring-system',
        custom_details: details
      }
    };

    try {
      const response = await axios.post(this.apiUrl, payload);
      return response.data.dedup_key;
    } catch (error) {
      console.error('Failed to create PagerDuty incident:', error);
      throw error;
    }
  }

  async resolveIncident(dedupKey) {
    const payload = {
      routing_key: this.integrationKey,
      event_action: 'resolve',
      dedup_key: dedupKey
    };

    await axios.post(this.apiUrl, payload);
  }

  async acknowledgeIncident(dedupKey) {
    const payload = {
      routing_key: this.integrationKey,
      event_action: 'acknowledge',
      dedup_key: dedupKey
    };

    await axios.post(this.apiUrl, payload);
  }
}

module.exports = PagerDutyClient;

// slack-notifier.js
const axios = require('axios');

class SlackNotifier {
  constructor(webhookUrl) {
    this.webhookUrl = webhookUrl;
  }

  async sendAlert(alert) {
    const color = this.getSeverityColor(alert.severity);

    const message = {
      text: `*${alert.severity.toUpperCase()}*: ${alert.summary}`,
      attachments: [{
        color,
        fields: [
          {
            title: 'Description',
            value: alert.description,
            short: false
          },
          {
            title: 'Service',
            value: alert.service,
            short: true
          },
          {
            title: 'Environment',
            value: alert.environment,
            short: true
          },
          {
            title: 'Runbook',
            value: alert.runbookUrl,
            short: false
          }
        ],
        footer: 'Monitoring System',
        ts: Math.floor(Date.now() / 1000)
      }]
    };

    try {
      await axios.post(this.webhookUrl, message);
    } catch (error) {
      console.error('Failed to send Slack notification:', error);
    }
  }

  getSeverityColor(severity) {
    const colors = {
      critical: '#FF0000',
      warning: '#FFA500',
      info: '#0000FF'
    };
    return colors[severity] || '#808080';
  }
}

module.exports = SlackNotifier;