require "json"

#Loading previous distilled logs information into an array
dist_logs_filename = 'distilled-logs.json'
dist_logs = []
if not File.exist?(dist_logs_filename)
  puts "File " + dist_logs_filename + " missing. Creating a new one " + Time.now.to_s
else
  puts "Reading " + dist_logs_filename + " " + Time.now.to_s
  dist_logs_raw = File.read(dist_logs_filename)
  dist_logs = JSON.parse(dist_logs_raw)
  puts dist_logs.size.to_s + " entries loaded"
end

blurt_only_samples = false
blurt_only_samples = true if dist_logs.size > 5000


#Look at log files and refresh/add to the distilled info where necessary
puts Time.now.to_s + " Running ls command"
$stdout.flush

muddles = {"20120715T124716.log" => [[35.0,-8.0], [35.0,-6.0]],
           "20120715T140025.log" => [[37.0,17.0], [38.0,-12.0]],
           "20120715T140332.log" => [[37.0,11.0], [37.0,11.0]],
           "20120715T140342.log" => [[38.0,-7.0], [38.0,-7.0]],
           "20120716T102735.log" => [[35.0,11.0], [44.0,11.0]],
           "20120716T102925.log" => [[45.0,3.0], [45.0,3.0]],
           "20120716T102955.log" => [[46.0,-13.0], [46.0,-13.0]],
           "20120716T103004.log" => [[46.0,-10.0], [46.0,-8.0]],      
           "20120716T103013.log" => [[46.0,-11.0], [46.0,-9.0], [46.0,-7.0]],
           "20120716T103022.log" => [[46.0,-3.0], [46.0,-3.0]],
           "20120716T103025.log" => [[46.0,-6.0], [46.0,-6.0]],
           "20120716T103034.log" => [[46.0,11.0], [46.0,11.0]],
           "20120716T103050.log" => [[46.0,-4.0], [46.0,-4.0]],
           "20120716T103059.log" => [[46.0,15.0], [46.0,15.0]],
           "20120716T104155.log" => [[46.0,16.0], [46.0,16.0]],
           "20120717T095414.log" => [[54.0,8.0], [54.0,16.0]],
           "20120717T095626.log" => [[56.0,-8.0], [56.0,2.0]] }

became_misaligned = false
read_count = 0  
line_no = -1
%x[ls -l /home/gravitystorm/public_html/redactions/logs-live --time-style=full-iso].each do |line|

  #parse the ls output
  line_bits = line.split(" ")
  size      = line_bits[4].to_i
  
  if line_bits[0]=="total"
    puts line
   
    
  elsif size>150
  
    
    mod_date  = line_bits[5].to_s
    mod_time  = line_bits[6].split('.')[0]
    filename  = line_bits[8]
                                                
    modified = mod_date + " " + mod_time
    dist_modified = ""

    line_no +=1
    
    #break if line_no==200
                      
    dist_log = []    
    if line_no < dist_logs.length and not became_misaligned
      dist_log = dist_logs[line_no]
      
      if dist_log.nil?
        dist_log = [ ] 
      else 	      
      
        dist_filename = dist_log[0]
        dist_modified = dist_log[1]
        
        if dist_filename!=filename
          puts "array misalignment " + dist_filename + "!=" + filename + " at line " + line_no.to_s
          became_misaligned = true
          dist_log = [ ]

        elsif dist_modified < modified
          #We need to scan this file and rewrite the dist_log entry. it's newly modified 
          dist_log = [ ]
        end
      end 
    end
               
    if dist_log.empty? 
      if line_no % 2000 == 0
        puts "Distilling the log entry for file " + filename + " (" +  line_no.to_s + " of " + dist_log.size.to_s + ")"
        $stdout.flush
      end
      #We're either rewriting a dist_log entry or making a new one
      read_count +=1          
      head = `head /home/gravitystorm/public_html/redactions/logs-live/#{filename}`
     
      region_line = ""          
      head.lines.each do |head_line|
        region_line = head_line if head_line.include?('Processing region')
      end
     
      if region_line!="" 
      	       
        region_hash = region_line.split("region {")[1]
        region_hash.gsub!(":","")
        region_hash.gsub!("=>",":")
        chomped_string = region_hash.split("lat:")[1]
        lat = chomped_string.split(", ")[0]
        chomped_string = chomped_string.split("lon:")[1]
        lon = chomped_string.split("}")[0]
       
        status = "UNKNOWN"
        end_time = ""
        tail = `tail /home/gravitystorm/public_html/redactions/logs-live/#{filename}`                           
        if tail.include?(': Summary')
          tail = tail.split(': Summary')[1]
          summary = ""
          successful_changesets = true
          successful_candidates = true
          failed_changesets = true     
          failed_candidates = true
          tail.lines.each do | tailline |
            if tailline.include?('-- :')
              summary += tailline.split('-- : ')[1].strip + "<br>"
              successful_changesets = false if tailline.include?(" 0 successful changesets")
              successful_candidates = false if tailline.include?(" 0 successful candidates")
              failed_changesets = false if tailline.include?(" 0 failed changesets")
              failed_candidates = false if tailline.include?(" 0 failed candidates")
            end
          end                                
                                                      
                                                           
          status = "SUCCESS"                        
          status = "ZEROS" if not successful_changesets and not successful_candidates
          status = "ERRORS" if failed_changesets or failed_candidates
        elsif tail.include?('Marking region failed')
          status="FAIL"
          summary="Region failed"
        end 
        
        start_time = ""
        start_time = head.split("Logfile created on ")[1] if head.include?("Logfile created on ")
        start_time = start_time.split(" by logger.rb")[0] if start_time.include?(" by logger.rb")
        
        last_line = tail.split("\n").last
        end_time = ""
        end_time = last_line.split('.')[0] if last_line.include?('.')
        end_time = end_time.split('I, [')[1] if end_time.include?('I, [')    
       
        description = "Region (" + lat + ", " + lon + ")<br>" +      
        	             "Started time:" + start_time + "<br>" +   
        	             "Ended time:" + end_time + "<br><br>" +    
                      summary.to_s + "<br><br>" +
                      "Detailed logs: <a href='http://gravitystorm.dev.openstreetmap.org/redactions/logs-live/" +
                      filename + "'>" + filename + "</a>"
       
       
        if status=="UNKNOWN" and summary.nil?
          status="CURRENT" 
          description = "No summary yet. This region appears to be 'in progress'.<br><br>" + description
        end 
       
       
        dist_log[0] = filename
        dist_log[1] = modified
        dist_log[2] = status
        dist_log[3] = start_time
        dist_log[4] = end_time
        dist_log[5] = lat
        dist_log[6] = lon
        dist_log[7] = description 
        
        dist_logs[line_no] = dist_log #write values back to the array
        
      
      end #endif
    end #endif
  end #endif
  
end #next line of the command output

raise("something went wrong with distilled logs. zero size") if dist_logs.empty?

puts "Writing back " + dist_logs.size.to_s + " distilled logs to file " + dist_logs_filename + " " + Time.now.to_s
File.open(dist_logs_filename,"w") do |f|
  f.write(dist_logs.to_json.gsub("\],\[", "\],\n\["))
end


puts "Building regions " + Time.now.to_s
$stdout.flush

regions = {}
ordering = [ ]

dist_logs.each do |dist_log|
  if dist_log.nil?
    #skip. happens sometimes during dev
  else 		
    filename    = dist_log[0]
    modified    = dist_log[1]
    status      = dist_log[2]
    start_time  = dist_log[3]
    end_time    = dist_log[4]
    lat         = dist_log[5]
    lon         = dist_log[6]
    description = dist_log[7]
	
    
    if not muddles[filename].nil?
      other_regions = [ ]  
      status = "MUDDLED"
      description = "This log file got muddled. It contains multiple regions:<br><br>"
      muddles[filename].each do |muddled_reg|
        description += "[" + muddled_reg[0].to_s + "," + muddled_reg[1].to_s + "] "
      end
      description += "<br><br>(Probably successful but it confused Harry's log processing)<br><br>"
      description += "Detailed logs: <a href='http://gravitystorm.dev.openstreetmap.org/redactions/logs-live/" +
                      filename + "'>" + filename + "</a>"

      muddles[filename].each do |muddled_reg|
        lat = muddled_reg[0]
        lon = muddled_reg[1]
        
        latlon = lat.to_s + "," + lon.to_s
      
        if regions[latlon].nil?
          #We didn't encounter these coords before. New region
          regions[latlon] = [lat, lon, status, description]; 
          ordering << latlon
        else 
          #We encountered these coords before. Modify this region data
          regions[latlon][3] = description + "<br><hr>...and an earlier run:<br>" + regions[latlon][3];
          regions[latlon][2] = status
          regions[latlon][2] = "RERUNS" if status=="SUCCESS" or status=="ZEROS"
       
          ordering.delete(latlon) 
          ordering << latlon
        end #endif
      end
      
    else 
      #A normal non-muddled log
      
      latlon = lat.to_s + "," + lon.to_s
      
      if regions[latlon].nil?
        #We didn't encounter these coords before. New region
        regions[latlon] = [lat, lon, status, description]; 
        ordering << latlon
      else 
        #We encountered these coords before. Modify this region data
        regions[latlon][3] = description + "<br><hr>...and an earlier run:<br>" + regions[latlon][3];
        regions[latlon][2] = status
        regions[latlon][2] = "RERUNS" if status=="SUCCESS" or status=="ZEROS"
     
        ordering.delete(latlon) 
        ordering << latlon
      end #endif
    
    end #endif
   
  end #endif  
end #next dist_log


puts "Adding todo regions"
$stdout.flush
todocount = 0
for lon in -180..179
  for lat in -90..89
    latlon = lat.to_s + ".0," + lon.to_s + ".0"
    if regions[latlon].nil?
      todocount +=1
      #puts latlon + " is still todo"
      regions[latlon] = [lat, lon, "TODO", "Still do to<br>(Region [" + lat.to_s + "," + lon.to_s + "])"]; 
      ordering << latlon
    end
  end
end
puts todocount.to_s + " regions todo still"

botprocessing_filename = 'botprocessingwriting.json'
botprocessingfile = File.open(botprocessing_filename, 'w') 

nosuccess_filename = 'nosuccess.json'
nosuccessfile = File.open(nosuccess_filename, 'w') 

botprocessingfile.write("regions=[")
nosuccessfile.write("regions=[")

ordering.each_with_index do |latlon, i|
  region = regions[latlon] 
  status = region[2]
  
  #add quotes
  region[2] = '"' + region[2] + '"'
  region[3] = '"' + region[3] + '"'

  puts i.to_s + " of " + regions.size.to_s + " e.g. " + region[0..2].join(",") if i % 5000==0 or i==regions.size-1

  outputline = "[" + region.join(",") + "]"
  outputline += ",\n" unless i==regions.size-1
  
  botprocessingfile.write(outputline)
  
  nosuccessfile.write(outputline) if status!="SUCCESS" and status!="RERUNS" and status!="ZEROS"
    
end
botprocessingfile.write("];")
nosuccessfile.write("];")

puts "Written " + regions.size.to_s + " regions to " + botprocessing_filename
puts "Written fewer regions to " + nosuccess_filename
puts "DONE " + Time.now.to_s
$stdout.flush