Project

General

Profile

Download (21.1 KB) Statistics
| Branch: | Tag: | Revision:

hammer-cli-import / lib / hammer_cli_import / base.rb @ ff3ed47d

1
#
2
# Copyright (c) 2014 Red Hat Inc.
3
#
4
# This file is part of hammer-cli-import.
5
#
6
# hammer-cli-import is free software: you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation, either version 3 of the License, or
9
# (at your option) any later version.
10
#
11
# hammer-cli-import is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
18
#
19

    
20
require 'csv'
21
require 'json'
22
require 'set'
23

    
24
require 'apipie-bindings'
25
require 'hammer_cli'
26

    
27
module HammerCLIImport
28
  class MissingObjectError < RuntimeError
29
  end
30

    
31
  class ImportRecoveryError < RuntimeError
32
  end
33

    
34
  class BaseCommand < HammerCLI::Apipie::Command
35
    extend PersistentMap::Extend
36
    extend ImportTools::ImportLogging::Extend
37
    extend AsyncTasksReactor::Extend
38

    
39
    include PersistentMap::Include
40
    include ImportTools::ImportLogging::Include
41
    include ImportTools::Task::Include
42
    include ImportTools::Exceptional::Include
43
    include AsyncTasksReactor::Include
44

    
45
    def initialize(*list)
46
      super(*list)
47

    
48
      # wrap API parameters into extra hash
49
      @wrap_out = {
50
        :users => :user,
51
        :template_snippets => :config_template
52
      }
53
      # APIs return objects encapsulated in extra hash
54
      #@wrap_in = {:organizations => 'organization'}
55
      @wrap_in = {}
56
      # entities that needs organization to be listed
57
      @prerequisite = {
58
        :activation_keys => :organizations,
59
        :content_views => :organizations,
60
        :content_view_versions => :organizations,
61
        :host_collections => :organizations,
62
        :products => :organizations,
63
        :repositories => :organizations,
64
        :repository_sets => :products,
65
        :hosts => :organizations
66
      }
67
      # cache imported objects (created/lookuped)
68
      @cache = {}
69
      class << @cache
70
        def []=(key, val)
71
          raise "@cache: #{val.inspect} is not a hash!" unless val.is_a? Hash
72
          super
73
        end
74
      end
75
      @summary = {}
76
      # Initialize AsyncTaskReactor
77
      atr_init
78

    
79
      server = (HammerCLI::Settings.settings[:_params] &&
80
                 HammerCLI::Settings.settings[:_params][:host]) ||
81
        HammerCLI::Settings.get(:foreman, :host)
82
      username = (HammerCLI::Settings.settings[:_params] &&
83
                   HammerCLI::Settings.settings[:_params][:username]) ||
84
        HammerCLI::Settings.get(:foreman, :username)
85
      password = (HammerCLI::Settings.settings[:_params] &&
86
                  HammerCLI::Settings.settings[:_params][:password]) ||
87
        HammerCLI::Settings.get(:foreman, :password)
88
      @api = ApipieBindings::API.new({
89
                                       :uri => server,
90
                                       :username => username,
91
                                       :password => password,
92
                                       :api_version => 2
93
                                     })
94
    end
95

    
96
    # What spacewalk-report do we expect to use for a given subcommand
97
    class << self; attr_accessor :reportname end
98

    
99
    option ['--csv-file'], 'FILE_NAME', 'CSV file with data to be imported', :required => true \
100
    do |filename|
101
      raise ArgumentError, "File #{filename} does not exist" unless File.exist? filename
102
      missing = CSVHelper.csv_missing_columns filename, self.class.csv_columns
103
      raise ArgumentError, "Bad CSV file #{filename}, missing columns: #{missing.inspect}" unless missing.empty?
104
      filename
105
    end
106

    
107
    option ['--delete'], :flag, 'Delete entities from CSV file', :default => false
108

    
109
    # TODO: Implement logic for verify
110
    # option ['--verify'], :flag, 'Verify entities from CSV file'
111

    
112
    option ['--recover'], 'RECOVER', 'Recover strategy, can be: rename (default), map, none', :default => :rename \
113
    do |strategy|
114
      raise ArgumentError, "Unknown '#{strategy}' strategy argument." \
115
        unless [:rename, :map, :none].include? strategy.to_sym
116
      strategy.to_sym
117
    end
118
    add_logging_options
119

    
120
    class << self
121
      # Which columns have to be be present in CSV.
122
      def csv_columns(*list)
123
        return @csv_columns if list.empty?
124
        raise 'set more than once' if @csv_columns
125
        @csv_columns = list
126
      end
127
    end
128

    
129
    class << self
130
      # Initialize API. Needed to be called before any +api_call+ calls.
131
      # If used in shell, it may be called multiple times
132
      def api_init
133
        @api = HammerCLIForeman.foreman_api_connection.api
134
        nil
135
      end
136

    
137
      # Call API. Ideally accessed via +api_call+ instance method.
138
      # This is supposed to be the only way to access @api.
139
      def api_call(resource, action, params = {}, headers = {}, dbg = false)
140
        if resource == :organizations && action == :create
141
          params[:organization] ||= {}
142
          params[:organization][:name] = params[:name]
143
        end
144
        @api.resource(resource).call(action, params, headers)
145
      rescue
146
        error("Error on api.resource(#{resource.inspect}).call(#{action.inspect}, #{params.inspect}):") if dbg
147
        raise
148
      end
149
    end
150

    
151
    # Call API. Convenience method for calling +api_call+ class method.
152
    def api_call(*list)
153
      self.class.api_call(*list)
154
    end
155

    
156
    # Call API on corresponding resource (defined by +map_target_entity+).
157
    def mapped_api_call(entity_type, *list)
158
      api_call(map_target_entity[entity_type], *list)
159
    end
160

    
161
    def data_dir
162
      File.join(File.expand_path('~'), '.transition_data')
163
    end
164

    
165
    # This method is called to process single CSV line when
166
    # importing.
167
    def import_single_row(_row)
168
      error 'Import not implemented.'
169
    end
170

    
171
    # This method is called to process single CSV line when
172
    # deleting
173
    def delete_single_row(_row)
174
      error 'Delete not implemented.'
175
    end
176

    
177
    def get_cache(entity_type)
178
      @cache[map_target_entity[entity_type]]
179
    end
180

    
181
    def load_cache
182
      maps.collect { |map_sym| map_target_entity[map_sym] } .uniq.each do |entity_type|
183
        list_server_entities entity_type
184
      end
185
    end
186

    
187
    def lookup_entity(entity_type, entity_id, online_lookup = false)
188
      if (!get_cache(entity_type)[entity_id] || online_lookup)
189
        get_cache(entity_type)[entity_id] = mapped_api_call(entity_type, :show, {'id' => entity_id})
190
      else
191
        debug "#{to_singular(entity_type).capitalize} #{entity_id} taken from cache."
192
      end
193
      return get_cache(entity_type)[entity_id]
194
    end
195

    
196
    def was_translated(entity_type, import_id)
197
      return @pm[entity_type].to_hash.value?(import_id)
198
    end
199

    
200
    def _compare_hash(entity_hash, search_hash)
201
      equal = nil
202
      search_hash.each do |key, value|
203
        if value.is_a? Hash
204
          equal = _compare_hash(entity_hash[key], search_hash[key])
205
        else
206
          equal = entity_hash[key] == value
207
        end
208
        return false unless equal
209
      end
210
      return true
211
    end
212

    
213
    def lookup_entity_in_cache(entity_type, search_hash)
214
      get_cache(entity_type).each do |_entity_id, entity_hash|
215
        return entity_hash if _compare_hash(entity_hash, search_hash)
216
      end
217
      return nil
218
    end
219

    
220
    def lookup_entity_in_array(array, search_hash)
221
      return nil if array.nil?
222
      array.each do |entity_hash|
223
        return entity_hash if _compare_hash(entity_hash, search_hash)
224
      end
225
      return nil
226
    end
227

    
228
    def last_in_cache?(entity_type, id)
229
      return get_cache(entity_type).size == 1 && get_cache(entity_type).first[0] == id
230
    end
231

    
232
    # Method for use when writing messages to user.
233
    #     > to_singular(:contentveiws)
234
    #     "contentview"
235
    #     > to_singular(:repositories)
236
    #     "repository"
237
    def to_singular(plural)
238
      return plural.to_s.gsub(/_/, ' ').sub(/s$/, '').sub(/ie$/, 'y')
239
    end
240

    
241
    def split_multival(multival, convert_to_int = true, separator = ';')
242
      arr = (multival || '').split(separator).delete_if { |v| v == 'None' }
243
      arr.map!(&:to_i) if convert_to_int
244
      return arr
245
    end
246

    
247
    # Method to call when you have created/deleted/found/mapped... something.
248
    # Collected data used for summary reporting.
249
    #
250
    # :found is used for situation, when you want to create something,
251
    # but you found out, it is already created.
252
    def report_summary(verb, item)
253
      raise "Not summary supported action: #{verb}" unless
254
        [:created, :deleted, :found, :mapped, :skipped, :uploaded, :wrote, :failed].include? verb
255
      @summary[verb] ||= {}
256
      @summary[verb][item] = @summary[verb].fetch(item, 0) + 1
257
    end
258

    
259
    def print_summary
260
      progress 'Summary'
261
      @summary.each do |verb, what|
262
        what.each do |entity, count|
263
          noun = if count == 1
264
                   to_singular entity
265
                 else
266
                   entity
267
                 end
268
          report = "  #{verb.to_s.capitalize} #{count} #{noun}."
269
          if verb == :found
270
            info report
271
          else
272
            progress report
273
          end
274
        end
275
      end
276
      progress '  No action taken.' if (@summary.keys - [:found]).empty?
277
    end
278

    
279
    def get_translated_id(entity_type, entity_id)
280
      if @pm[entity_type] && @pm[entity_type][entity_id]
281
        return @pm[entity_type][entity_id]
282
      end
283
      raise MissingObjectError, "Unable to import. Please import the #{to_singular(entity_type)} with id [ #{entity_id.inspect} ] then try again."
284
    end
285

    
286
    # this method returns a *first* found original_id
287
    # (since we're able to map several organizations into one)
288
    def get_original_id(entity_type, import_id)
289
      if was_translated(entity_type, import_id)
290
        # find original_ids
291
        @pm[entity_type].to_hash.each do |key, value|
292
          return key if value == import_id
293
        end
294
      else
295
        debug "Unknown imported #{to_singular(entity_type)} [#{import_id}]."
296
      end
297
      return nil
298
    end
299

    
300
    def list_server_entities(entity_type, extra_hash = {}, use_cache = false)
301
      if @prerequisite[entity_type]
302
        list_server_entities(@prerequisite[entity_type]) unless @cache[@prerequisite[entity_type]]
303
      end
304

    
305
      @cache[entity_type] ||= {}
306
      results = []
307

    
308
      if !extra_hash.empty? || @prerequisite[entity_type].nil?
309
        if use_cache
310
          @list_cache ||= {}
311
          if @list_cache[entity_type]
312
            return @list_cache[entity_type][extra_hash] if @list_cache[entity_type][extra_hash]
313
          else
314
            @list_cache[entity_type] ||= {}
315
          end
316
        end
317
        entities = api_call(entity_type, :index, {'per_page' => 999999}.merge(extra_hash))
318
        results = entities['results']
319
        @list_cache[entity_type][extra_hash] = results if use_cache
320
      elsif @prerequisite[entity_type] == :organizations
321
        # check only entities in imported orgs (not all of them)
322
        @pm[:organizations].to_hash.values.each do |org_id|
323
          entities = api_call(entity_type, :index, {'per_page' => 999999, 'organization_id' => org_id})
324
          results += entities['results']
325
        end
326
      else
327
        @cache[@prerequisite[entity_type]].each do |pre_id, _|
328
          entities = api_call(
329
            entity_type,
330
            :index,
331
            {
332
              'per_page' => 999999,
333
              @prerequisite[entity_type].to_s.sub(/s$/, '_id').to_sym => pre_id
334
            })
335
          results += entities['results']
336
        end
337
      end
338

    
339
      results.each do |entity|
340
        entity['id'] = entity['id'].to_s if entity_type == :hosts
341
        @cache[entity_type][entity['id']] = entity
342
      end
343
    end
344

    
345
    def map_entity(entity_type, original_id, id)
346
      if @pm[entity_type][original_id]
347
        info "#{to_singular(entity_type).capitalize} [#{original_id}->#{@pm[entity_type][original_id]}] already mapped. " \
348
          'Skipping.'
349
        report_summary :found, entity_type
350
        return
351
      end
352
      info "Mapping #{to_singular(entity_type)} [#{original_id}->#{id}]."
353
      @pm[entity_type][original_id] = id
354
      report_summary :mapped, entity_type
355
      return get_cache(entity_type)[id]
356
    end
357

    
358
    def unmap_entity(entity_type, target_id)
359
      deleted = @pm[entity_type].delete_value(target_id)
360
      info " Unmapped #{to_singular(entity_type)} with id #{target_id}: #{deleted}x" if deleted > 1
361
    end
362

    
363
    def find_uniq(arr)
364
      uniq = nil
365
      uniq = arr[0] if arr[1].is_a?(Array) &&
366
                       (arr[1][0] =~ /has already been taken/ ||
367
                        arr[1][0] =~ /already exists/ ||
368
                        arr[1][0] =~ /must be unique within one organization/)
369
      return uniq
370
    end
371

    
372
    def found_errors(err)
373
      return err && err['errors'] && err['errors'].respond_to?(:each)
374
    end
375

    
376
    def recognizable_error(arr)
377
      return arr.is_a?(Array) && arr.size >= 2
378
    end
379

    
380
    def process_error(err, entity_hash)
381
      uniq = nil
382
      err['errors'].each do |arr|
383
        next unless recognizable_error(arr)
384
        uniq = find_uniq(arr)
385
        break if uniq && entity_hash.key?(uniq.to_sym)
386
        uniq = nil # otherwise uniq is not usable
387
      end
388
      return uniq
389
    end
390

    
391
    # Create entity, with recovery strategy.
392
    #
393
    # * +:map+ - Use existing entity
394
    # * +:rename+ - Change name
395
    # * +nil+ - Fail
396
    def create_entity(entity_type, entity_hash, original_id, recover = nil, retries = 2)
397
      raise ImportRecoveryError, "Creation of #{entity_type} not recovered by " \
398
        "'#{recover || option_recover.to_sym}' strategy" if retries < 0
399
      uniq = nil
400
      begin
401
        return _create_entity(entity_type, entity_hash, original_id)
402
      rescue RestClient::UnprocessableEntity => ue
403
        error " Creation of #{to_singular(entity_type)} failed."
404
        uniq = nil
405
        err = JSON.parse(ue.response)
406
        err = err['error'] if err.key?('error')
407
        if found_errors(err)
408
          uniq = process_error(err, entity_hash)
409
        end
410
        raise ue unless uniq
411
      end
412

    
413
      uniq = uniq.to_sym
414

    
415
      case recover || option_recover.to_sym
416
      when :rename
417
        entity_hash[uniq] = original_id.to_s + '-' + entity_hash[uniq]
418
        info " Recovering by renaming to: \"#{uniq}\"=\"#{entity_hash[uniq]}\""
419
        return create_entity(entity_type, entity_hash, original_id, recover, retries - 1)
420
      when :map
421
        entity = lookup_entity_in_cache(entity_type, {uniq.to_s => entity_hash[uniq]})
422
        if entity
423
          info " Recovering by remapping to: #{entity['id']}"
424
          return map_entity(entity_type, original_id, entity['id'])
425
        else
426
          warn "Creation of #{entity_type} not recovered by \'#{recover}\' strategy."
427
          raise ImportRecoveryError, "Creation of #{entity_type} not recovered by \'#{recover}\' strategy."
428
        end
429
      else
430
        fatal 'No recover strategy.'
431
        raise ue
432
      end
433
      nil
434
    end
435

    
436
    # Use +create_entity+ instead.
437
    def _create_entity(entity_type, entity_hash, original_id)
438
      type = to_singular(entity_type)
439
      if @pm[entity_type][original_id]
440
        info type.capitalize + ' [' + original_id.to_s + '->' + @pm[entity_type][original_id].to_s + '] already imported.'
441
        report_summary :found, entity_type
442
        return get_cache(entity_type)[@pm[entity_type][original_id]]
443
      else
444
        info 'Creating new ' + type + ': ' + entity_hash.values_at(:name, :label, :login).compact[0]
445
        if entity_type == :hosts
446
          entity = @api.resource(:host_subscriptions).call(:create, entity_hash)
447
          params = {
448
            'id' => entity['id'],
449
            'host' => {
450
              'comment' => entity_hash[:description]
451
            }
452
          }
453
          entity = @api.resource(:hosts).call(:update, params)
454
          unless entity_hash[:host_collection_ids].empty?
455
            @api.resource(:host_collections).call(:add_hosts, {
456
                'id' => entity_hash[:host_collection_ids][0],
457
                'host_ids' => [entity['id']]
458
            })
459
          end
460
          entity['id'] = entity['id'].to_s
461
        else
462
          entity_hash = {@wrap_out[entity_type] => entity_hash} if @wrap_out[entity_type]
463
          debug "entity_hash: #{entity_hash.inspect}"
464
          entity = mapped_api_call(entity_type, :create, entity_hash)
465
        end
466
        debug "created entity: #{entity.inspect}"
467
        entity = entity[@wrap_in[entity_type]] if @wrap_in[entity_type]
468
        @pm[entity_type][original_id] = entity['id']
469
        get_cache(entity_type)[entity['id']] = entity
470
        debug "@pm[#{entity_type}]: #{@pm[entity_type].inspect}"
471
        report_summary :created, entity_type
472
        return entity
473
      end
474
    end
475

    
476
    def update_entity(entity_type, id, entity_hash)
477
      info "Updating #{to_singular(entity_type)} with id: #{id}"
478
      mapped_api_call(entity_type, :update, {:id => id}.merge!(entity_hash))
479
    end
480

    
481
    # Delete entity by original (Sat5) id
482
    def delete_entity(entity_type, original_id)
483
      type = to_singular(entity_type)
484
      unless @pm[entity_type][original_id]
485
        error 'Unknown ' + type + ' to delete [' + original_id.to_s + '].'
486
        return nil
487
      end
488
      info 'Deleting imported ' + type + ' [' + original_id.to_s + '->' + @pm[entity_type][original_id].to_s + '].'
489
      begin
490
        mapped_api_call(entity_type, :destroy, {:id => @pm[entity_type][original_id]})
491
        # delete from cache
492
        get_cache(entity_type).delete(@pm[entity_type][original_id])
493
        # delete from pm
494
        unmap_entity(entity_type, @pm[entity_type][original_id])
495
        report_summary :deleted, entity_type
496
      rescue => e
497
        warn "Delete of #{to_singular(entity_type)} [#{original_id}] failed with #{e.class}: #{e.message}"
498
        report_summary :failed, entity_type
499
      end
500
    end
501

    
502
    # Delete entity by target (Sat6) id
503
    def delete_entity_by_import_id(entity_type, import_id, delete_key = 'id')
504
      type = to_singular(entity_type)
505
      original_id = get_original_id(entity_type, import_id)
506
      if original_id.nil?
507
        error 'Unknown imported ' + type + ' to delete [' + import_id.to_s + '].'
508
        return nil
509
      end
510
      info "Deleting imported #{type} [#{original_id}->#{@pm[entity_type][original_id]}]."
511
      if delete_key == 'id'
512
        delete_id = import_id
513
      else
514
        delete_id = get_cache(entity_type)[import_id][delete_key]
515
      end
516
      begin
517
        mapped_api_call(entity_type, :destroy, {:id => delete_id})
518
        # delete from cache
519
        get_cache(entity_type).delete(import_id)
520
        # delete from pm
521
        @pm[entity_type].delete original_id
522
        report_summary :deleted, entity_type
523
      rescue => e
524
        warn "Delete of #{to_singular(entity_type)} [#{delete_id}] failed with #{e.class}: #{e.message}"
525
        report_summary :failed, entity_type
526
      end
527
    end
528

    
529
    # Wait for asynchronous task.
530
    #
531
    # * +uuid+ - UUID of async task.
532
    # * +start_wait+ - Seconds to wait before first check.
533
    # * +delta_wait+ - How much longer will every next wait be (unless +max_wait+ is reached).
534
    # * +max_wait+ - Maximum time to wait between two checks.
535
    def wait_for_task(uuid, start_wait = 0, delta_wait = 1, max_wait = 10)
536
      wait_time = start_wait
537
      if option_quiet?
538
        info "Waiting for the task [#{uuid}] "
539
      else
540
        print "Waiting for the task [#{uuid}] "
541
      end
542

    
543
      loop do
544
        sleep wait_time
545
        wait_time = [wait_time + delta_wait, max_wait].min
546
        print '.' unless option_quiet?
547
        STDOUT.flush unless option_quiet?
548
        task = api_call(:foreman_tasks, :show, {:id => uuid})
549
        next unless task['state'] == 'stopped'
550
        print "\n" unless option_quiet?
551
        return task['return'] == 'success'
552
      end
553
    end
554

    
555
    def cvs_iterate(filename, action)
556
      CSVHelper.csv_each filename, self.class.csv_columns do |data|
557
        handle_missing_and_supress "processing CSV line:\n#{data.inspect}" do
558
          action.call(data)
559
        end
560
      end
561
    end
562

    
563
    def import(filename)
564
      cvs_iterate(filename, (method :import_single_row))
565
    end
566

    
567
    def post_import(_csv_file)
568
      # empty by default
569
    end
570

    
571
    def post_delete(_csv_file)
572
      # empty by default
573
    end
574

    
575
    def delete(filename)
576
      cvs_iterate(filename, (method :delete_single_row))
577
    end
578

    
579
    def execute
580
      # Get set up to do logging as soon as reasonably possible
581
      setup_logging
582
      # create a storage directory if not exists yet
583
      Dir.mkdir data_dir unless File.directory? data_dir
584

    
585
      # initialize apipie binding
586
      self.class.api_init
587
      load_persistent_maps
588
      load_cache
589
      prune_persistent_maps @cache
590
      # TODO: This big ugly thing might need some cleanup
591
      begin
592
        if option_delete?
593
          info "Deleting from #{option_csv_file}"
594
          delete option_csv_file
595
          handle_missing_and_supress 'post_delete' do
596
            post_delete option_csv_file
597
          end
598
        else
599
          info "Importing from #{option_csv_file}"
600
          import option_csv_file
601
          handle_missing_and_supress 'post_import' do
602
            post_import option_csv_file
603
          end
604
        end
605
        atr_exit
606
      rescue StandardError, SystemExit, Interrupt => e
607
        error "Exiting: #{e}"
608
        logtrace e
609
      end
610
      save_persistent_maps
611
      print_summary
612
      HammerCLI::EX_OK
613
    end
614
  end
615
end
616
# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby