Plugins » Katello

My "Sync Repository on Capsule(s)" tasks regularly fail with:

PLP0034: The distributor tkc-holdings-puppet-puppet-forge_puppet indicated a failed response when publishing repository tkc-holdings-puppet-puppet-forge.

CentOS 7
Foreman 1.13.4
Katello 3.2.2
Pulp 2.9.3

Output and backtrace are below.

When this happens I have to manually sync up /var/lib/pulp/content/units/puppet_module between the two servers and resume the task.

Output:

{"pulp_tasks"=>
  [{"exception"=>nil,
    "task_type"=>"pulp.server.managers.repo.sync.sync",
    "_href"=>"/pulp/api/v2/tasks/6bf168f6-8217-46aa-bcd5-d6928f89085d/",
    "task_id"=>"6bf168f6-8217-46aa-bcd5-d6928f89085d",
    "tags"=>
     ["pulp:repository:tkc-holdings-puppet-puppet-forge", "pulp:action:sync"],
    "finish_time"=>"2017-04-13T13:20:39Z",
    "_ns"=>"task_status",
    "start_time"=>"2017-04-13T11:42:05Z",
    "traceback"=>nil,
    "spawned_tasks"=>
     [{"_href"=>"/pulp/api/v2/tasks/66df8bd1-13b7-4d00-b4f8-d35515d19914/",
       "task_id"=>"66df8bd1-13b7-4d00-b4f8-d35515d19914"}],
    "progress_report"=>
     {"puppet_importer"=>
       {"modules"=>
         {"error_message"=>nil,
          "execution_time"=>5909,
          "total_count"=>5478,
          "traceback"=>nil,
          "individual_errors"=>[],
          "state"=>"success",
          "error_count"=>0,
          "error"=>"None",
          "finished_count"=>5478},
        "metadata"=>
         {"query_finished_count"=>1,
          "traceback"=>nil,
          "execution_time"=>3,
          "query_total_count"=>1,
          "error_message"=>nil,
          "state"=>"success",
          "error"=>"None",
          "current_query"=>
           "https://sl1mmgplsat0001.ipa.gen.zone/pulp/puppet/tkc-holdings-puppet-puppet-forge/modules.json"}}},
    "queue"=>"reserved_resource_worker-3@om1mmgplcap0001.ipa.gen.zone.dq",
    "state"=>"finished",
    "worker_name"=>"reserved_resource_worker-3@om1mmgplcap0001.ipa.gen.zone",
    "result"=>
     {"result"=>"success",
      "importer_id"=>"puppet_importer",
      "exception"=>nil,
      "repo_id"=>"tkc-holdings-puppet-puppet-forge",
      "traceback"=>nil,
      "started"=>"2017-04-13T11:42:05Z",
      "_ns"=>"repo_sync_results",
      "completed"=>"2017-04-13T13:20:39Z",
      "importer_type_id"=>"puppet_importer",
      "error_message"=>nil,
      "summary"=>{"total_execution_time"=>5912},
      "added_count"=>5478,
      "removed_count"=>0,
      "updated_count"=>0,
      "id"=>"58ef7b2760e04fceab15ee23",
      "details"=>
       {"finished_count"=>5478, "total_count"=>5478, "error_count"=>0}},
    "error"=>nil,
    "_id"=>{"$oid"=>"58ef640de38a98313cf672c3"},
    "id"=>"58ef640de38a98313cf672c3"},
   {"exception"=>nil,
    "task_type"=>"pulp.server.managers.repo.publish.publish",
    "_href"=>"/pulp/api/v2/tasks/66df8bd1-13b7-4d00-b4f8-d35515d19914/",
    "task_id"=>"66df8bd1-13b7-4d00-b4f8-d35515d19914",
    "tags"=>
     ["pulp:repository:tkc-holdings-puppet-puppet-forge",
      "pulp:action:publish"],
    "finish_time"=>"2017-04-13T13:27:38Z",
    "_ns"=>"task_status",
    "start_time"=>"2017-04-13T13:27:17Z",
    "traceback"=>
     "Traceback (most recent call last):\n  File \"/usr/lib/python2.7/site-packages/celery/app/trace.py\", line 240, in trace_task\n    R = retval = fun(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/async/tasks.py\", line 484, in __call__\n    return super(Task, self).__call__(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/async/tasks.py\", line 103, in __call__\n    return super(PulpTask, self).__call__(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/celery/app/trace.py\", line 437, in __protected_call__\n    return self.run(*args, **kwargs)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/controllers/repository.py\", line 971, in publish\n    result = check_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/controllers/repository.py\", line 1038, in check_publish\n    result = _do_publish(repo_obj, dist_id, dist_inst, transfer_repo, conduit, call_config)\n  File \"/usr/lib/python2.7/site-packages/pulp/server/controllers/repository.py\", line 1096, in _do_publish\n    distributor_id=dist_id, summary=publish_report.summary\nPulpCodedException: The distributor tkc-holdings-puppet-puppet-forge_puppet indicated a failed response when publishing repository tkc-holdings-puppet-puppet-forge.\n",
    "spawned_tasks"=>[],
    "progress_report"=>
     {"tkc-holdings-puppet-puppet-forge_puppet"=>
       {"modules"=>
         {"error_message"=>nil,
          "execution_time"=>19,
          "total_count"=>5203,
          "traceback"=>nil,
          "individual_errors"=>nil,
          "state"=>"success",
          "error_count"=>0,
          "error"=>"None",
          "finished_count"=>5203},
        "publishing"=>{"http"=>"not-started", "https"=>"not-started"},
        "metadata"=>
         {"execution_time"=>1,
          "state"=>"failed",
          "error_message"=>"Error generating repository metadata",
          "error"=>
           "[Errno 2] No such file or directory: u'/var/lib/pulp/content/units/puppet_module/8e/f1fdc06f71f636c389d165bec906b17e64d2ca08804fb17f97d0ed6df40db4/jbarrajon-nexus-0.1.0.tar.gz'",
          "traceback"=>
           [["/usr/lib/python2.7/site-packages/pulp_puppet/plugins/distributors/publish.py",
             143,
             "_metadata_step",
             "self._generate_dependency_data(modules)"],
            ["/usr/lib/python2.7/site-packages/pulp_puppet/plugins/distributors/publish.py",
             319,
             "_generate_dependency_data",
             "with open(module._storage_path, 'rb') as file_handle:"]]}}},
    "queue"=>"reserved_resource_worker-3@om1mmgplcap0001.ipa.gen.zone.dq",
    "state"=>"error",
    "worker_name"=>"reserved_resource_worker-3@om1mmgplcap0001.ipa.gen.zone",
    "result"=>nil,
    "error"=>
     {"code"=>"PLP0034",
      "data"=>
       {"distributor_id"=>"tkc-holdings-puppet-puppet-forge_puppet",
        "repo_id"=>"tkc-holdings-puppet-puppet-forge",
        "summary"=>{"total_execution_time"=>20}},
      "description"=>
       "The distributor tkc-holdings-puppet-puppet-forge_puppet indicated a failed response when publishing repository tkc-holdings-puppet-puppet-forge.",
      "sub_errors"=>[]},
    "_id"=>{"$oid"=>"58ef7b27e38a98313cf68a96"},
    "id"=>"58ef7b27e38a98313cf68a96"}],
 "poll_attempts"=>{"total"=>406, "failed"=>1}}

Backtrace:

/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/pulp/abstract_async_task.rb:121:in `block in external_task='
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/pulp/abstract_async_task.rb:119:in `each'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/pulp/abstract_async_task.rb:119:in `external_task='
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/polling.rb:98:in `poll_external_task_with_rescue'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/polling.rb:21:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/cancellable.rb:9:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/pulp/abstract_async_task.rb:45:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:506:in `block (3 levels) in execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:30:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/remote_action.rb:16:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/remote_action.rb:40:in `block in as_remote_user'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/models/katello/concerns/user_extensions.rb:21:in `cp_config'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/remote_action.rb:27:in `as_cp_user'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/remote_action.rb:39:in `as_remote_user'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/remote_action.rb:16:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:30:in `with_progress_calculation'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:16:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:22:in `with_locale'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:30:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/world.rb:30:in `execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:505:in `block (2 levels) in execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in `catch'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in `block in execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `block in with_error_handling'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `catch'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `with_error_handling'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:499:in `execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:260:in `execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:9:in `block (2 levels) in execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in `with_meta_calculation'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:8:in `block in execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:22:in `open_action'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:7:in `execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:15:in `block in on_message'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in `block in assigns'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `tap'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `assigns'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:in `match_value'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in `block in match?'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `each'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `match?'
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:in `match'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:12:in `on_message'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/context.rb:46:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/executes_context.rb:7:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/actor.rb:26:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/awaits.rb:15:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:38:in `process_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:31:in `process_envelopes?'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:20:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/termination.rb:55:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/removes_child.rb:10:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in `on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:161:in `process_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:95:in `block in on_envelope'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:118:in `block (2 levels) in schedule_execution'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/synchronization/mri_lockable_object.rb:38:in `block in synchronize'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/synchronization/mri_lockable_object.rb:38:in `synchronize'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/synchronization/mri_lockable_object.rb:38:in `synchronize'
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:115:in `block in schedule_execution'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/serialized_execution.rb:18:in `call'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/serialized_execution.rb:18:in `call'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/serialized_execution.rb:96:in `work'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/serialized_execution.rb:77:in `block in call_job'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/ruby_thread_pool_executor.rb:333:in `call'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/ruby_thread_pool_executor.rb:333:in `run_task'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/ruby_thread_pool_executor.rb:322:in `block (3 levels) in create_worker'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/ruby_thread_pool_executor.rb:305:in `loop'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/ruby_thread_pool_executor.rb:305:in `block (2 levels) in create_worker'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/ruby_thread_pool_executor.rb:304:in `catch'
/opt/rh/sclo-ror42/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/executor/ruby_thread_pool_executor.rb:304:in `block in create_worker'
/opt/theforeman/tfm/root/usr/share/gems/gems/logging-1.8.2/lib/logging/diagnostic_context.rb:323:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/logging-1.8.2/lib/logging/diagnostic_context.rb:323:in `block in create_with_logging_context'