Project

General

Profile

Defect #8857 » 0001-Git-repo-parsing-optimization-without-db-query-group.patch

Gergely Fábián, 2012-03-08 21:28

View differences:

app/models/repository/git.rb
156 156
      from_scmid = nil
157 157
      from_scmid = h["branches"][br]["last_scmid"] if h["branches"][br]
158 158
      h["branches"][br] ||= {}
159

  
159
      
160 160
      revisions = scm.revisions('', from_scmid, br, {:reverse => true})
161 161
      next if revisions.blank?
162

  
162
      
163 163
      # Remember the last commit id here, before we start removing revisions from the array.
164 164
      # We'll do that for optimization, but it also means, that we may lose even all revisions.
165 165
      last_revision  = revisions.last
......
170 170
      # (this equals to a union, because we executed diff above)
171 171
      all_revisions += revisions.map{|r| r.scmid}
172 172

  
173
      # Make the search for existing revisions in the database in a more sufficient manner
174
      # This is replacing the one-after-one queries.
175
      # Find all revisions, that are in the database, and then remove them from the revision array.
176
      # Then later we won't need any conditions for db existence.
177
      # Query for several revisions at once, and remove them from the revisions array, if they are there.
178
      # Do this in chunks, to avoid eventual memory problems (in case of tens of thousands of commits).
179
      # If there are no revisions (because the original code's algoritm filtered them),
180
      # then this part will be stepped over.
181
      # We make queries, just if there is any revision.
182
      limit = 100
183
      offset = 0
184
      revisions_copy = revisions.clone # revisions will change
185
      while offset < revisions_copy.size
186
        recent_changesets_slice = changesets.find(
187
                                     :all,
188
                                     :conditions => [
189
                                        'scmid IN (?)',
190
                                        revisions_copy.slice(offset, limit).map{|x| x.scmid}
191
                                      ]
192
                                    )
193
        # Subtract revisions that redmine already knows about
194
        recent_revisions = recent_changesets_slice.map{|c| c.scmid}
195
        revisions.reject!{|r| recent_revisions.include?(r.scmid)}
196
        offset += limit
197
      end
198

  
199 173
      revisions.each do |rev|
200
        transaction do
201
          # There is no search in the db for this revision, because above we ensured,
202
          # that it's not in the db.
203
          db_saved_rev = save_revision(rev)
204
          parents = {}
205
          parents[db_saved_rev] = rev.parents unless rev.parents.nil?
206
          parents.each do |ch, chparents|
207
            ch.parents = chparents.collect{|rp| find_changeset_by_name(rp)}.compact
174
        db_rev = find_changeset_by_name(rev.revision)
175
        if db_rev.nil?
176
          transaction do
177
            db_saved_rev = save_revision(rev)
178
            parents = {}
179
            parents[db_saved_rev] = rev.parents unless rev.parents.nil?
180
            parents.each do |ch, chparents|
181
              ch.parents = chparents.collect{|rp| find_changeset_by_name(rp)}.compact
182
            end
208 183
          end
209 184
          # saving the last scmid was moved from here, because we won't come in here,
210 185
          # if the revision was already added for another branch
211 186
        end
212 187
      end
213

  
188
      
214 189
      # save the data about the last revision for this branch
215 190
      unless last_revision.nil?
216 191
        h["branches"][br]["last_scmid"] = last_revision.scmid
(6-6/8)