Project

General

Profile

Download (13.7 KB) Statistics
| Branch: | Tag: | Revision:

hammer-cli-csv / lib / hammer_cli_csv / splice.rb @ 611959a3

1
require 'openssl'
2
require 'date'
3

    
4
module HammerCLICsv
5
  class CsvCommand
6
    class SpliceCommand < BaseCommand
7
      command_name 'splice'
8
      desc         'import Satellite-5 splice data'
9

    
10
      option %w(--dir), 'DIR',
11
          'Directory of Splice exported CSV files (default pwd)'
12
      option %w(--mapping-dir), 'DIR',
13
          'Directory of Splice product mapping files (default /usr/share/rhsm/product/RHEL-6)'
14

    
15
      UUID = 'server_id'
16
      ORGANIZATION = 'organization'
17
      ORGANIZATION_ID = 'org_id'
18
      NAME = 'name'
19
      HOSTNAME = 'hostname'
20
      IP_ADDRESS = 'ip_address'
21
      IPV6_ADDRESS = 'ipv6_address'
22
      REGISTERED_BY = 'registered_by'
23
      REGISTRATION_TIME = 'registration_time'
24
      LAST_CHECKIN_TIME = 'last_checkin_time'
25
      PRODUCTS = 'software_channel'
26
      ENTITLEMENTS = 'entitlements'
27
      HOSTCOLLECTIONS = 'system_group'
28
      VIRTUAL_HOST = 'virtual_host'
29
      ARCHITECTURE = 'architecture'
30
      HARDWARE = 'hardware'
31
      MEMORY = 'memory'
32
      SOCKETS = 'sockets'
33
      IS_VIRTUALIZED = 'is_virtualized'
34

    
35
      def import
36
        @existing = {}
37
        load_product_mapping
38
        preload_host_guests
39

    
40
        filename = option_dir + '/splice-export'
41
        thread_import(false, filename, NAME) do |line|
42
          create_content_hosts_from_csv(line) unless line[UUID][0] == '#'
43
        end
44

    
45
        update_host_guests
46
        delete_unfound_hosts(@existing)
47
      end
48

    
49
      def create_content_hosts_from_csv(line)
50
        return if option_organization && line[ORGANIZATION] != option_organization
51

    
52
        if !@existing[line[ORGANIZATION]]
53
          create_organization(line)
54
          @existing[line[ORGANIZATION]] = {}
55

    
56
          # Fetching all content hosts is too slow and times out due to the complexity of the data
57
          # rendered in the json.
58
          # http://projects.theforeman.org/issues/6307
59
          total = @api.resource(:systems).call(:index, {
60
              'organization_id' => foreman_organization(:name => line[ORGANIZATION]),
61
              'per_page' => 1
62
          })['total'].to_i
63
          (total / 20 + 2).to_i.times do |page|
64
            @api.resource(:systems).call(:index, {
65
                'organization_id' => foreman_organization(:name => line[ORGANIZATION]),
66
                'page' => page,
67
                'per_page' => 20
68
            })['results'].each do |host|
69
              @existing[line[ORGANIZATION]][host['name']] = host['uuid'] if host
70
            end
71
          end
72
        end
73

    
74
        name = "#{line[NAME]}-#{line[UUID]}"
75
        #checkin_time = Time.parse(line[LAST_CHECKIN_TIME]).strftime("%a, %d %b %Y %H:%M:%S %z")
76
        checkin_time = if line[LAST_CHECKIN_TIME].casecmp('now').zero?
77
                         DateTime.now.strftime("%a, %d %b %Y %H:%M:%S %z")
78
                       else
79
                         DateTime.parse(line[LAST_CHECKIN_TIME]).strftime("%a, %d %b %Y %H:%M:%S %z")
80
                       end
81

    
82
        if !@existing[line[ORGANIZATION]].include? name
83
          print(_("Creating content host '%{name}'...") % {:name => name}) if option_verbose?
84
          host_id = @api.resource(:systems).call(:create, {
85
              'name' => name,
86
              'organization_id' => foreman_organization(:name => line[ORGANIZATION]),
87
              'environment_id' => lifecycle_environment(line[ORGANIZATION], :name => 'Library'),
88
              'content_view_id' => katello_contentview(line[ORGANIZATION], :name => 'Default Organization View'),
89
              'last_checkin' => checkin_time,
90
              'facts' => facts(name, line),
91
              'installed_products' => products(line),
92
              'type' => 'system'
93
          })['uuid']
94

    
95
          # last_checkin is not updated in candlepin on creation
96
          # https://bugzilla.redhat.com/show_bug.cgi?id=1212122
97
          @api.resource(:systems).call(:update, {
98
            'id' => host_id,
99
            'system' => {
100
                'last_checkin' => checkin_time
101
            },
102
            'last_checkin' => checkin_time
103
          })
104

    
105
        else
106
          print(_("Updating content host '%{name}'...") % {:name => name}) if option_verbose?
107
          host_id = @api.resource(:systems).call(:update, {
108
              'id' => @existing[line[ORGANIZATION]][name],
109
              'system' => {
110
                  'name' => name,
111
                  'environment_id' => lifecycle_environment(line[ORGANIZATION], :name => 'Library'),
112
                  'content_view_id' => katello_contentview(line[ORGANIZATION], :name => 'Default Organization View'),
113
                  'last_checkin' => checkin_time,
114
                  'facts' => facts(name, line),
115
                  'installed_products' => products(line)
116
              },
117
              'installed_products' => products(line),  # TODO: http://projects.theforeman.org/issues/9191
118
              'last_checkin' => checkin_time
119
          })['uuid']
120

    
121
          @existing[line[ORGANIZATION]].delete(name) # Remove to indicate found
122
        end
123

    
124
        if @hosts.include? line[UUID]
125
          @hosts[line[UUID]] = host_id
126
        elsif @guests.include? line[UUID]
127
          @guests[line[UUID]] = "#{line[ORGANIZATION]}/#{name}"
128
        end
129

    
130
        update_host_collections(host_id, line)
131

    
132
        puts _('done') if option_verbose?
133
      end
134

    
135
      private
136

    
137
      def facts(name, line)
138
        facts = {}
139
        facts['system.certificate_version'] = '3.2'  # Required for auto-attach to work
140
        facts['network.hostname'] = line[NAME]
141
        facts['network.ipv4_address'] = line[IP_ADDRESS]
142
        facts['network.ipv6_address'] = line[IPV6_ADDRESS]
143
        facts['memory.memtotal'] = line[MEMORY]
144
        facts['uname.machine'] = line[ARCHITECTURE]
145
        facts['virt.is_guest'] = line[IS_VIRTUALIZED] == 'Yes' ? true : false
146
        facts['virt.uuid'] = "#{line[ORGANIZATION]}/#{name}" if facts['virt.is_guest']
147

    
148
        # 1 CPUs 1 Sockets; eth0 10.11....
149
        hardware = line[HARDWARE].split(' ')
150
        if hardware[1] == 'CPUs'
151
          facts['cpu.cpu(s)'] = hardware[0] unless hardware[0] == 'unknown'
152
          facts['cpu.cpu_socket(s)'] = hardware[2] unless hardware[0] == 'unknown'
153
          # facts['cpu.core(s)_per_socket']  Not present in data
154
        end
155

    
156
        facts
157
      end
158

    
159
      def update_host_collections(host_id, line)
160
        return nil if !line[HOSTCOLLECTIONS]
161

    
162
        @existing_hostcollections ||= {}
163
        if @existing_hostcollections[line[ORGANIZATION]].nil?
164
          @existing_hostcollections[line[ORGANIZATION]] = {}
165
          @api.resource(:host_collections).call(:index, {
166
              :per_page => 999999,
167
              'organization_id' => foreman_organization(:name => line[ORGANIZATION])
168
          })['results'].each do |hostcollection|
169
            @existing_hostcollections[line[ORGANIZATION]][hostcollection['name']] = hostcollection['id']
170
          end
171
        end
172

    
173
        CSV.parse_line(line[HOSTCOLLECTIONS], {:col_sep => ';'}).each do |hostcollection_name|
174
          if @existing_hostcollections[line[ORGANIZATION]][hostcollection_name].nil?
175
            hostcollection_id = @api.resource(:host_collections).call(:create, {
176
                'organization_id' => foreman_organization(:name => line[ORGANIZATION]),
177
                'name' => hostcollection_name,
178
                'unlimited_content_hosts' => true,
179
                'max_content_hosts' => nil
180
            })['id']
181
            @existing_hostcollections[line[ORGANIZATION]][hostcollection_name] = hostcollection_id
182
          end
183

    
184
          @api.resource(:host_collections).call(:add_systems, {
185
              'id' => @existing_hostcollections[line[ORGANIZATION]][hostcollection_name],
186
              'system_ids' => [host_id]
187
          })
188
        end
189
      end
190

    
191
      def products(line)
192
        return nil if !line[PRODUCTS]
193
        products = CSV.parse_line(line[PRODUCTS], {:col_sep => ';'}).collect do |channel|
194
          product = @product_mapping[channel]
195
          if product.nil?
196
            # puts _("WARNING: No product found for channel '%{name}'") % {:name => channel}
197
            next
198
          end
199
          product
200
        end
201
        products.compact
202
      end
203

    
204
      def preload_host_guests
205
        @hosts = {}
206
        @guests = {}
207
        return unless option_dir && File.exists?(option_dir + "/host-guests")
208
        host_guest_file = option_dir + "/host-guests"
209

    
210
        CSV.foreach(host_guest_file, {
211
            :skip_blanks => true,
212
            :headers => :first_row,
213
            :return_headers => false
214
        }) do |line|
215
          @hosts[line['server_id']] = nil
216
          CSV.parse_line(line['guests'], {:col_sep => ';'}).each do |guest|
217
            @guests[guest] = nil
218
          end
219
        end
220
      end
221

    
222
      def update_host_guests
223
        return unless option_dir && File.exists?(option_dir + "/host-guests")
224
        return if @hosts.empty?
225
        host_guest_file = option_dir + "/host-guests"
226

    
227
        print _('Updating hypervisor and guest associations...') if option_verbose?
228

    
229
        CSV.foreach(host_guest_file, {
230
            :skip_blanks => true,
231
            :headers => :first_row,
232
            :return_headers => false
233
        }) do |line|
234
          host_id = @hosts[line['server_id']]
235
          next if host_id.nil?
236
          guest_ids = CSV.parse_line(line['guests'], {:col_sep => ';'}).collect do |guest|
237
            @guests[guest]
238
          end
239

    
240
          @api.resource(:systems).call(:update, {
241
              'id' => host_id,
242
              'guest_ids' => guest_ids
243
          })
244
        end
245

    
246
        puts _("done") if option_verbose?
247
      end
248

    
249
      def update_subscriptions(host_id, line)
250
        existing_subscriptions = @api.resource(:subscriptions).call(:index, {
251
            'organization_id' => foreman_organization(:name => line[ORGANIZATION]),
252
            'per_page' => 999999,
253
            'system_id' => host_id
254
        })['results']
255
        if existing_subscriptions.length > 0
256
          @api.resource(:subscriptions).call(:destroy, {
257
            'system_id' => host_id,
258
            'id' => existing_subscriptions[0]['id']
259
          })
260
        end
261

    
262
        return if line[SUBSCRIPTIONS].nil? || line[SUBSCRIPTIONS].empty?
263

    
264
        subscriptions = CSV.parse_line(line[SUBSCRIPTIONS], {:skip_blanks => true}).collect do |details|
265
          (amount, sku, name) = details.split('|')
266
          {
267
            :id => get_subscription(line[ORGANIZATION], :name => name),
268
            :quantity => (amount.nil? || amount.empty? || amount == 'Automatic') ? 0 : amount.to_i
269
          }
270
        end
271

    
272
        @api.resource(:subscriptions).call(:create, {
273
            'system_id' => host_id,
274
            'subscriptions' => subscriptions
275
        })
276
      end
277

    
278
      def create_organization(line)
279
        if !@existing_organizations
280
          @existing_organizations = {}
281
          @api.resource(:organizations).call(:index, {
282
              :per_page => 999999
283
          })['results'].each do |organization|
284
            @existing_organizations[organization['name']] = organization['id'] if organization
285
          end
286
        end
287

    
288
        if !@existing_organizations[line[ORGANIZATION]]
289
          print _("Creating organization '%{name}'... ") % {:name => line[ORGANIZATION]} if option_verbose?
290
          @api.resource(:organizations).call(:create, {
291
              'name' => line[ORGANIZATION],
292
              'organization' => {
293
                  'name' => line[ORGANIZATION],
294
                  'label' => "splice-#{line[ORGANIZATION_ID]}",
295
                  'description' => _('Satellite-5 Splice')
296
              }
297
          })
298
          puts _('done')
299
        end
300
      end
301

    
302
      def delete_unfound_hosts(hosts)
303
        hosts.keys.each do |organization|
304
          hosts[organization].values.each do |host_id|
305
            print _("Deleting content host with id '%{id}'...") % {:id => host_id}
306
            @api.resource(:systems).call(:destroy, {:id => host_id})
307
            puts _('done')
308
          end
309
        end
310
      end
311

    
312

    
313
      def load_product_mapping
314
        @product_mapping = {}
315

    
316
        mapping_dir = (option_mapping_dir || '/usr/share/rhsm/product/RHEL-6')
317
        File.open(mapping_dir + '/channel-cert-mapping.txt', 'r') do |file|
318
          file.each_line do |line|
319
            # '<product name>: <file name>\n'
320
            (product_name, file_name) = line.split(':')
321
            @product_mapping[product_name] = {:file => "#{mapping_dir}/#{file_name[1..-2]}"}
322
            OpenSSL::X509::Certificate.new(File.read(@product_mapping[product_name][:file])).extensions.each do |extension|
323
              if extension.oid.start_with?("1.3.6.1.4.1.2312.9.1.")
324
                oid_parts = extension.oid.split('.')
325
                @product_mapping[product_name][:productId] = oid_parts[-2].to_i
326
                case oid_parts[-1]
327
                when /1/
328
                  @product_mapping[product_name][:productName] = extension.value[2..-1] #.sub(/\A\.+/,'')
329
                when /2/
330
                  @product_mapping[product_name][:version] = extension.value[2..-1] #.sub(/\A\.+/,'')
331
                when /3/
332
                  @product_mapping[product_name][:arch] = extension.value[2..-1] #.sub(/\A\.+/,'')
333
                end
334
              end
335
            end
336
          end
337
        end
338

    
339
        channel_file = option_dir + '/cloned-channels'
340
        return unless File.exists? channel_file
341
        unmatched_channels = []
342
        CSV.foreach(channel_file, {
343
            :skip_blanks => true,
344
            :headers => :first_row,
345
            :return_headers => false
346
        }) do |line|
347
          if @product_mapping[line['original_channel_label']]
348
            @product_mapping[line['new_channel_label']] = @product_mapping[line['original_channel_label']]
349
          else
350
            unmatched_channels << line
351
          end
352
        end
353

    
354
        # Second pass through
355
        unmatched_channels.each do |line|
356
          next if @product_mapping[line['original_channel_label']].nil?
357
          @product_mapping[line['new_channel_label']] = @product_mapping[line['original_channel_label']]
358
        end
359
      end
360
    end
361
  end
362
end