Analytics backup fails with error: unable to call remote backup endpoint - invalid status code 500

kubectl logs ibm-apiconnect-<id>_ibm-apiconnect -n <namespace where APIC operator is deployed>

{"level":"info","ts":"2024-11-25T14:15:13.989Z","logger":"controllers.AnalyticsBackup","msg":"calling analytics director backup status endpoint","analyticsbackup":{"name":"analytics-backup-01","namespace":"cp4i"},"id":"apic-dev-a7s-all-2024-11-25t14:08:15.265z"}
{"level":"info","ts":"2024-11-25T14:15:13.989Z","logger":"controllers.AnalyticsBackup","msg":"doHTTPWithQuery","analyticsbackup":{"name":"analytics-backup-01","namespace":"cp4i"},"a7sClient.analyticsDirectorServerUrl":"https://apic-dev-a7s-director.cp4i:3009","httpMethod":"GET","urlPath":"backups/local/apic-dev-a7s-all-2024-11-25t14:08:15.265z/status","urlQuery":""}
{"level":"info","ts":"2024-11-25T14:15:14.018Z","logger":"controllers.AnalyticsBackup","msg":"statusCheck","analyticsbackup":{"name":"analytics-backup-01","namespace":"cp4i"},"request":"https://apic-dev-a7s-director.cp4i:3009/backups/local/apic-dev-a7s-all-2024-11-25t14:08:15.265z/status","response code":200}
{"level":"info","ts":"2024-11-25T14:15:14.018Z","logger":"controllers.AnalyticsBackup","msg":"processHTTPResponse","analyticsbackup":{"name":"analytics-backup-01","namespace":"cp4i"},"responseBytes":"{\"trace\":\"705a139d-6312-4c3b-920b-fec683e3b4cb\",\"status_code\":200,\"id\":\"apic-dev-a7s-all-2024-11-25t14:08:15.265z\",\"status\":\"SUCCESS\"}"}
{"level":"info","ts":"2024-11-25T14:15:14.018Z","logger":"controllers.AnalyticsBackup","msg":"calling analytics director remote backup endpoint","analyticsbackup":{"name":"analytics-backup-01","namespace":"cp4i"}}
{"level":"info","ts":"2024-11-25T14:15:14.018Z","logger":"controllers.AnalyticsBackup","msg":"doHTTPWithQuery","analyticsbackup":{"name":"analytics-backup-01","namespace":"cp4i"},"a7sClient.analyticsDirectorServerUrl":"https://apic-dev-a7s-director.cp4i:3009","httpMethod":"POST","urlPath":"backups/remote","urlQuery":""}
{"level":"info","ts":"2024-11-25T14:15:14.031Z","logger":"controllers.AnalyticsBackup","msg":"statusCheck","analyticsbackup":{"name":"analytics-backup-01","namespace":"cp4i"},"request":"https://apic-dev-a7s-director.cp4i:3009/backups/remote","response code":500}
{
  "level": "error",
  "ts": "2024-11-25T14:15:14.031Z",
  "logger": "controllers.AnalyticsBackup",
  "msg": "an error occurred triggering the remote backup",
  "analyticsbackup": {
    "name": "analytics-backup-01",
    "namespace": "cp4i"
  },
  "error": "unable to call remote backup endpoint: invalid status code 500: {\"trace\":\"c1927f11-2744-4820-9ed5-1055d19edf26\",\"status_code\":500,\"errors\":[{\"code\":\"too_long_http_line_exception\",\"message\":\"An unexpected error occurred attempting to get index sizes\",\"more_info\":\"An HTTP line is larger than 4096 bytes.\",\"target\":{}}]}",
  "stacktrace": "github.ibm.com/velox/apiconnect-operator/ibm-apiconnect/controllers/analytics%2eapiconnect.(*AnalyticsBackupReconciler).reconcileRemoteBackup\n\t/workspace/ibm-apiconnect/controllers/analytics.apiconnect/analyticsbackup_controller.go:324\ngithub.ibm.com/velox/apiconnect-operator/ibm-apiconnect/controllers/analytics%2eapiconnect.(*AnalyticsBackupReconciler).Reconcile\n\t/workspace/ibm-apiconnect/controllers/analytics.apiconnect/analyticsbackup_controller.go:233\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile\n\t/tmp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:119\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/tmp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/tmp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/tmp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:227"
}
{
  "level": "error",
  "ts": "2024-11-25T14:15:14.037Z",
  "msg": "Reconciler error",
  "controller": "analyticsbackup",
  "controllerGroup": "analytics.apiconnect.ibm.com",
  "controllerKind": "AnalyticsBackup",
  "AnalyticsBackup": {
    "name": "analytics-backup-01",
    "namespace": "cp4i"
  },
  "namespace": "cp4i",
  "name": "analytics-backup-01",
  "reconcileID": "2ac2d8dc-23a3-435a-b1f7-c5cd26ef0274",
  "error": "unable to call remote backup endpoint: invalid status code 500: {\"trace\":\"c1927f11-2744-4820-9ed5-1055d19edf26\",\"status_code\":500,\"errors\":[{\"code\":\"too_long_http_line_exception\",\"message\":\"An unexpected error occurred attempting to get index sizes\",\"more_info\":\"An HTTP line is larger than 4096 bytes.\",\"target\":{}}]}",
  "stacktrace": "sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/tmp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:329\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/tmp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/tmp/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.0/pkg/internal/controller/controller.go:227"
}

kubectl logs apic-dev-a7s-director-<id>_director -n <namespace where APIC analytics subsystem is deployed>

2024-11-25 14:18:17.489 [info] 0e20a200-6d15-41e6-a622-0c1413b54427 Found repository apic-analytics
2024-11-25 14:18:17.490 [error] 0e20a200-6d15-41e6-a622-0c1413b54427 An unexpected error occurred attempting to get index sizese [ResponseError]: too_long_http_line_exception: [too_long_http_line_exception] Reason: An HTTP line is larger than 4096 bytes.
    at onBody (/node_modules/@opensearch-project/opensearch/lib/Transport.js:426:23)
    at err (/node_modules/@opensearch-project/opensearch/lib/Transport.js:341:11)
    at IncomingMessage.emit (node:events:531:35)
    at endReadableNT (node:internal/streams/readable:1696:12)
    at process.processTicksAndRejections (node:internal/process/task_queues:82:21) {
  meta: {
    body: { error: [Object], status: 400 },
    statusCode: 400,
    headers: {
      'content-type': 'application/json; charset=UTF-8',
      'content-length': '219'
    },
    meta: {
      context: null,
      request: [Object],
      name: 'opensearch-js',
      connection: [Object],
      attempts: 0,
      aborted: false
    }
  }
}