upload_external_src_gorongoza.rb

require 'csv'

#prod
project = Project.find(593)
zoo_user = User.find(1)
subject_set = SubjectSet.find(681)

#staging
# project = Project.find(450)
# zoo_user = User.find(27)
# subject_set = SubjectSet.find(2725)

#cam's dev box
# subject_set = SubjectSet.find(2435)

if project && zoo_user && subject_set
  rows = CSV.foreach("#{Rails.root}/hhmi-23k.csv", headers: true).to_a
  rows.in_groups_of(500) do |group|
    subjects = []
    failed_subjects = []

    subject_group = group.map do |row|
      header, external_src = row.delete("FileName")
      ext = external_src.split(".").last
      metadata = row.to_hash
      content_type = case ext
      when /jpg/i
        "image/jpeg"
      else
        nil
      end

      if external_src.nil? || content_type.nil? || metadata.empty?
        raise "No src / content type from file"
      end

      {external_src: external_src, metadata: metadata, content_type: content_type}
    end

    urls = subject_group.map {|i| i[:external_src] }

    existing_subjects = Subject.joins(:locations).where(project_id: project.id, upload_user_id: zoo_user.id,
                                                        media: {external_link: true, src: urls}).load

    subject_group.each do |item|
      subject = existing_subjects.find {|i| i.locations.map(&:src).include?(item[:external_src])}

      if subject
        subjects << subject
      else
        subject = Subject.new(project_id: project.id, upload_user_id: zoo_user.id, metadata: item[:metadata])
        location_params = { content_type: item[:content_type], external_link: true, src: item[:external_src], metadata: { index: 0 }}
        subject.locations.build(location_params)

        if subject.save
          subjects << subject
        else
          failed_subjects << subject
        end
      end
    end

    #cleanup if failure
    unless failed_subjects.empty?
      # subjects.map(&:destroy)
      puts "failed to create some fo the subjects..."
      #check these here
      binding.pry
    end

    #now link to the subject sets
    subject_ids_to_link = SetMemberSubject.where(subject_set_id: subject_set.id).where.not(subject_id: subjects.map(&:id)).pluck(:subject_id)

    if Subject.where(id: subject_ids_to_link).count != subject_ids_to_link.size
      raise "Error: check the subject set and all the subjects exist."
    end
    new_sms_values = subject_ids_to_link.map do |subject_id|
      [ subject_set.id, subject_id, rand ]
    end
    sset_import_cols = %w(subject_set_id subject_id random)
    SetMemberSubject.import sset_import_cols, new_sms_values, validate: false
    SubjectSet.reset_counters(subject_set.id, :set_member_subjects)
    puts "Created #{subjects.size} subjects"
  end
else
  puts "fail - couldn't find a project, user or subject set to link to"
end