Project

General

Profile

0001-Fixes-405-and-Fixes-349-Adds-support-to-2.6.x-report.patch

Ohad Levy, 10/20/2010 05:09 PM

View differences:

app/models/log.rb
1
class Log < ActiveRecord::Base
2
  belongs_to :message
3
  belongs_to :source
4
  belongs_to :report
5
  validates_presence_of :message_id, :source_id, :report_id, :level_id
6

  
7
  LEVELS = [:debug, :info, :notice, :warning, :err, :alert, :emerg, :crit]
8

  
9
  def to_s
10
    "#{source} #{message}"
11
  end
12

  
13
  def level= l
14
    write_attribute(:level_id, LEVELS.index(l))
15
  end
16

  
17
  def level
18
    LEVELS[level_id]
19
  end
20

  
21
end
app/models/message.rb
1
class Message < ActiveRecord::Base
2
  has_many :reports, :through => :logs
3
  has_many :logs
4
  validates_presence_of :value
5

  
6
  def to_s
7
    value
8
  end
9

  
10
end
app/models/report.rb
1 1
class Report < ActiveRecord::Base
2 2
  belongs_to :host
3
  serialize :log, Puppet::Transaction::Report
4
  validates_presence_of :log, :host_id, :reported_at, :status
3
  has_many :messages, :through => :logs, :dependent => :destroy
4
  has_many :sources, :through => :logs, :dependent => :destroy
5
  has_many :logs, :dependent => :destroy
6
  validates_presence_of :host_id, :reported_at, :status
5 7
  validates_uniqueness_of :reported_at, :scope => :host_id
6 8

  
7 9
  METRIC = %w[applied restarted failed failed_restarts skipped]
......
41 43
    return type.nil? ? h : h[type]
42 44
  end
43 45

  
46
  # extracts serialized metrics and keep them as a hash_with_indifferent_access
47
  def metrics
48
    YAML.load(read_attribute(:metrics)).with_indifferent_access
49
  end
50

  
51
  # serialize metrics as YAML
52
  def metrics= m
53
    write_attribute(:metrics,m.to_yaml) unless m.nil?
54
  end
55

  
44 56
  # generate dynamically methods for all metrics
45 57
  # e.g. Report.last.applied
46 58
  METRIC.each do |method|
......
64 76
  end
65 77

  
66 78
  def config_retrieval
67
    t = validate_meteric("time", :config_retrieval)
68
    t.round_with_precision(2) if t
79
    metrics[:time][:config_retrieval].round_with_precision(2)
69 80
  end
70 81

  
71 82
  def runtime
72
    t = validate_meteric("time", :total)
73
    t.round_with_precision(2) if t
83
    (metrics[:time][:total] || metrics[:time].values.sum).round_with_precision(2)
74 84
  end
75 85

  
76 86
  #imports a YAML report into database
......
83 93

  
84 94
      # parse report metrics
85 95
      raise "Invalid report: can't find metrics information for #{report.host} at #{report.id}" if report.metrics.nil?
96

  
97
      # Is this a pre 2.6.x report format?
98
      @pre26 = !report.instance_variables.include?("@resource_statuses")
99

  
86 100
      # convert report status to bit field
87 101
      st = calc_status(metrics_to_hash(report))
88 102

  
......
101 115
      host.save_with_validation(false)
102 116

  
103 117
      # and save our report
104
      self.create! :host => host, :reported_at => report.time.utc, :log => report, :status => st
105

  
118
      r = self.create!(:host => host, :reported_at => report.time.utc, :status => st, :metrics => self.m2h(report.metrics))
119
      # Store all Puppet message logs
120
      r.import_log_messages report
121
      return r
106 122
    rescue Exception => e
107 123
      logger.warn "Failed to process report for #{report.host} due to:#{e}"
108 124
      false
......
142 158
    status = conditions[:status]
143 159
    cond = "created_at < \'#{(Time.now.utc - timerange).to_formatted_s(:db)}\'"
144 160
    cond += " and status = #{status}" unless status.nil?
145
    # delete the reports
146
    count = Report.delete_all(cond)
161
    # delete the reports, must use destroy_all vs. delete_all because of assoicated logs and METRIC
162
    count = Report.destroy_all(cond)
147 163
    logger.info Time.now.to_s + ": Expired #{count} Reports"
148 164
    return count
149 165
  end
......
160 176
    counter
161 177
  end
162 178

  
163
  protected
179
  def import_log_messages report
180
    report.logs.each do |r|
181
      message = Message.find_or_create_by_value r.message
182
      source  = Source.find_or_create_by_value r.source
183
      log = Log.create :message_id => message.id, :source_id => source.id, :report_id => self.id, :level => r.level
184
      log.errors.empty?
185
    end
186
  end
187

  
188
  private
164 189

  
165 190
  # Converts metrics form Puppet report into a hash
166 191
  # this hash is required by the calc_status method
167 192
  def self.metrics_to_hash(report)
168
    resources = report.metrics["resources"]
169 193
    report_status = {}
194
    metrics = report.metrics.with_indifferent_access
170 195

  
171 196
    # find our metric values
172
    METRIC.each { |m| report_status[m] = resources[m.to_sym] }
197
    METRIC.each do |m|
198
      if @pre26
199
        report_status[m] = metrics["resources"][m]
200
      else
201
        h=translate_metrics_to26(m)
202
        report_status[m] = metrics[h[:type]][h[:name]]
203
      end
204
      report_status[m] ||= 0
205
    end
206

  
173 207
    # special fix for false warning about skips
174 208
    # sometimes there are skip values, but there are no error messages, we ignore them.
175 209
    if report_status["skipped"] > 0 and ((report_status.values.sum) - report_status["skipped"] == report.logs.size)
176 210
      report_status["skipped"] = 0
177
    end
211
    end unless report_status["skipped"].nil?
178 212
    return report_status
179 213
  end
180 214

  
215

  
216
  # return all metrics as a hash
217
  def self.m2h metrics
218
    h = {}
219
    metrics.each do |title, mtype|
220
      h[mtype.name] ||= {}
221
      mtype.values.each{|m| h[mtype.name].merge!({m[0] => m[2]})}
222
    end
223
    return h
224
  end
225

  
226

  
181 227
  # converts a hash into a bit field
182 228
  # expects a metrics_to_hash kind of hash
183 229
  def self.calc_status (hash = {})
......
196 242
    nil
197 243
  end
198 244

  
245

  
246
  # The metrics layout has changed in Puppet 2.6.x release,
247
  # this method attempts to align the bit value metrics and the new name scheme in 2.6.x
248
  # returns a hash of { :type => "metric type", :name => "metric_name"}
249
  def self.translate_metrics_to26 metric
250
   case metric
251
    when "applied"
252
      { :type => "changes", :name => :total}
253
    else
254
      { :type => "resources", :name => metric.to_sym}
255
    end
256
  end
257

  
199 258
end
app/models/source.rb
1
class Source < ActiveRecord::Base
2
  has_many :reports, :through => :logs
3
  has_many :logs
4
  validates_presence_of :value
5

  
6
  def to_s
7
    value
8
  end
9
end
app/views/host_mailer/error_state.text.html.erb
5 5
    <title>Puppet error from <%= @host.to_label %></title>
6 6
  </head>
7 7
  <body style="background-color: #ffffff;">
8
    <%= render :partial => 'reports/output', :locals => {:logs => @report.log.logs } %>
8
    <%= render :partial => 'reports/output', :locals => {:logs => @report.logs } %>
9 9
  </body>
10 10
</html>
app/views/reports/show.rhtml
1
<% title @report.host.name %>
1
<% title @report.host.name -%>
2 2
Reported at <%= @report.reported_at.getlocal %>, which is <b><%= time_ago_in_words(@report.reported_at) %> ago</b>
3 3
<% if @offset > 100 -%>
4 4
  <div class="flash error">
......
9 9
  </div>
10 10
<% end -%>
11 11

  
12
<% if @report.log.logs.size > 0 -%>
13
  <%= render :partial => 'output', :locals => { :logs => @report.log.logs} %>
12
<% if @report.logs.size > 0 -%>
13
  <%= render :partial => 'output', :locals => { :logs => @report.logs} %>
14 14
<% end -%>
15 15

  
16 16
<div class="flash">
......
19 19
      <td> <b>Metrics</b></td>
20 20
      <td>
21 21
        <table style="width:100%;">
22
          <% @report.log.metrics["time"].values.each do |name, label, value|-%>
23
            <% if label == 'Total' then -%>
22
          <% @report.metrics["time"].each do |label, value|-%>
23
            <% if label.to_s =~ /total/i then -%>
24 24
              <%   @totaltime = value -%>
25 25
              <%   next -%>
26 26
            <% end -%>
......
29 29
              <td> <%= value.round_with_precision(4) rescue "N/A"%> </td>
30 30
            </tr>
31 31
          <% end %>
32
          <tr><td class="last_row">Total</td><td class="last_row"><%= h @totaltime.round_with_precision(4) rescue "N/A"%></td></tr>
32
          <tr><td class="last_row">Total</td><td class="last_row"><%= h (@totaltime || @report.runtime).round_with_precision(4) rescue "N/A"%></td></tr>
33 33
        </table>
34 34
      </td>
35 35
    </tr>
db/migrate/20101018120548_create_messages.rb
1
class CreateMessages < ActiveRecord::Migration
2
  def self.up
3
    create_table :messages do |t|
4
      t.text :value
5
    end
6
    if ActiveRecord::Base.connection.instance_values["config"][:adapter] == "mysql"
7
      execute "ALTER TABLE messages ENGINE = MYISAM"
8
      execute "ALTER TABLE messages ADD FULLTEXT (value)"
9
    else
10
      add_index :messages, :value
11
    end
12
  end
13

  
14
  def self.down
15
    remove_index :messages, :value
16
    drop_table :messages
17
  end
18
end
db/migrate/20101018120603_create_sources.rb
1
class CreateSources < ActiveRecord::Migration
2
  def self.up
3
    create_table :sources do |t|
4
      t.text :value
5
    end
6
    if ActiveRecord::Base.connection.instance_values["config"][:adapter] == "mysql"
7
      execute "ALTER TABLE sources ENGINE = MYISAM"
8
      execute "ALTER TABLE sources ADD FULLTEXT (value)"
9
    else
10
      add_index :sources, :value
11
    end
12
  end
13

  
14
  def self.down
15
    remove_index :sources, :value
16
    drop_table :sources
17
  end
18
end
db/migrate/20101018120621_create_logs.rb
1
class CreateLogs < ActiveRecord::Migration
2
  def self.up
3
    create_table :logs do |t|
4
      t.integer :source_id
5
      t.integer :message_id
6
      t.integer :report_id
7
      t.integer :level_id
8

  
9
      t.timestamps
10
    end
11
    add_index :logs, :report_id
12
    add_index :logs, :message_id
13
    add_index :logs, :level_id
14
  end
15

  
16
  def self.down
17
    remove_index :logs, :level_id
18
    remove_index :logs, :report_id
19
    remove_index :logs, :message_id
20
    drop_table :logs
21
  end
22
end
db/migrate/20101019122857_add_metrics_to_report.rb
1
class AddMetricsToReport < ActiveRecord::Migration
2
  def self.up
3
    add_column :reports, :metrics, :text
4
  end
5

  
6
  def self.down
7
    remove_column :reports, :metrics
8
  end
9
end
db/migrate/20101019183859_convert_reports.rb
1
class ConvertReports < ActiveRecord::Migration
2
  def self.up
3
    say "About to convert all of the #{Report.count} reports log field into a more DB optimized way... this might take a while....."
4

  
5
    Report.all.each do |report|
6
      case report.log.class.to_s
7
      when "Puppet::Transaction::Report"
8
        log = report.log
9
      when "String"
10
        log = YAML.load(report.log)
11
      else
12
        # this report might have been processed already, skipping
13
        next
14
      end
15

  
16
      # Is this a pre 2.6.x report format?
17
      pre26 = !report.instance_variables.include?("@resource_statuses")
18

  
19
      # Recalcuate the status field if this report is from a 2.6.x puppet client
20
      report.status = Report.calc_status(Report.metrics_to_hash(log)) unless pre26
21
      report.metrics = Report.m2h(log.metrics).with_indifferent_access
22

  
23
      report.import_log_messages(log)
24
      report.log = "" # not really needed, but this way the db can reuse some space instead of claim new one.
25

  
26
      report.save
27
    end
28
    remove_column :reports, :log
29

  
30
  end
31

  
32
  def self.down
33
    add_column :reports, :log, :text
34
    say "cant recreate the data, import it again"
35
  end
36
end
test/fixtures/logs.yml
1
# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
2

  
3
one:
4
  source_id: 1
5
  message_id: 1
6
  report_id: 1
7

  
8
two:
9
  source_id: 1
10
  message_id: 1
11
  report_id: 1
test/fixtures/messages.yml
1
# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
2

  
3
one:
4
  value: MyText
5

  
6
two:
7
  value: MyText
test/fixtures/sources.yml
1
# Read about fixtures at http://ar.rubyonrails.org/classes/Fixtures.html
2

  
3
one:
4
  value: MyText
5

  
6
two:
7
  value: MyText
test/unit/log_test.rb
1
require 'test_helper'
2

  
3
class LogTest < ActiveSupport::TestCase
4
  # Replace this with your real tests.
5
  test "the truth" do
6
    assert true
7
  end
8
end
test/unit/message_test.rb
1
require 'test_helper'
2

  
3
class MessageTest < ActiveSupport::TestCase
4
  # Replace this with your real tests.
5
  test "the truth" do
6
    assert true
7
  end
8
end
test/unit/source_test.rb
1
require 'test_helper'
2

  
3
class SourceTest < ActiveSupport::TestCase
4
  # Replace this with your real tests.
5
  test "the truth" do
6
    assert true
7
  end
8
end
0
-