<a name="280" href="#280">280</a>             logger.finer(<span class="string">"Adding "</span> + curi.toString());<a name="281" href="#281">281</a>         }<a name="282" href="#282">282</a>         <strong>try</strong>{<a name="283" href="#283">283</a>             <strong>if</strong>(inProcessing(curi.toString())){<a name="284" href="#284">284</a>                 <em class="comment">// If it is currently being processed, then it is already been </em><a name="285" href="#285">285</a>                 <em class="comment">// added and we sure as heck can't fetch it any sooner!</em><a name="286" href="#286">286</a>                 <strong>return</strong>;<a name="287" href="#287">287</a>             }<a name="288" href="#288">288</a>             <a name="289" href="#289">289</a>             OperationStatus opStatus = strictAdd(curi,false);<a name="290" href="#290">290</a>             <a name="291" href="#291">291</a>             <strong>long</strong> curiProcessingTime = curi.getLong(<a name="292" href="#292">292</a>                     A_TIME_OF_NEXT_PROCESSING);<a name="293" href="#293">293</a>     <a name="294" href="#294">294</a>             <strong>if</strong> (opStatus == OperationStatus.KEYEXIST){ <a name="295" href="#295">295</a>                 <em class="comment">// Override an existing URI</em><a name="296" href="#296">296</a>                 <em class="comment">// We need to extract the old CrawlURI (it contains vital</em><a name="297" href="#297">297</a>                 <em class="comment">// info on past crawls), check its scheduling directive</em><a name="298" href="#298">298</a>                 <em class="comment">// and (possibly) its time of next fetch and update if it</em><a name="299" href="#299">299</a>                 <em class="comment">// will promote the URI to an earlier processing time.</em><a name="300" href="#300">300</a>                 <strong>boolean</strong> update = false;<a name="301" href="#301">301</a>                 <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curiExisting = getCrawlURI(curi.toString());<a name="302" href="#302">302</a>                 <strong>long</strong> oldCuriProcessingTime = curiExisting.getLong(<a name="303" href="#303">303</a>                         A_TIME_OF_NEXT_PROCESSING);<a name="304" href="#304">304</a>                 <strong>if</strong>(curi.getSchedulingDirective() &lt; <a name="305" href="#305">305</a>                         curiExisting.getSchedulingDirective()){<a name="306" href="#306">306</a>                     <em class="comment">// New scheduling directive is of higher importance,</em><a name="307" href="#307">307</a>                     <em class="comment">// update to promote URI.</em><a name="308" href="#308">308</a>                     curiExisting.setSchedulingDirective(<a name="309" href="#309">309</a>                             curi.getSchedulingDirective());<a name="310" href="#310">310</a>                     update = <strong>true</strong>;<a name="311" href="#311">311</a>                 }<a name="312" href="#312">312</a>                 <strong>if</strong>( (curiProcessingTime &lt; oldCuriProcessingTime)<a name="313" href="#313">313</a>                         &amp;&amp; (overrideSetTimeOnDups || update)){<a name="314" href="#314">314</a>                     <em class="comment">// We update the processing time if it is earlier then </em><a name="315" href="#315">315</a>                     <em class="comment">// the original and either overrideSetTimeOnDups was set</em><a name="316" href="#316">316</a>                     <em class="comment">// or update is true, meaning a higher priority scheduling</em><a name="317" href="#317">317</a>                     <em class="comment">// directive for this URI.</em><a name="318" href="#318">318</a>                     curiExisting.putLong(<a name="319" href="#319">319</a>                             A_TIME_OF_NEXT_PROCESSING,<a name="320" href="#320">320</a>                             curiProcessingTime);<a name="321" href="#321">321</a>                     update = <strong>true</strong>;<a name="322" href="#322">322</a>                 }<a name="323" href="#323">323</a>                 <strong>if</strong>(update){<a name="324" href="#324">324</a>                     opStatus = strictAdd(curiExisting,<strong>true</strong>); <em class="comment">//Override</em><a name="325" href="#325">325</a>                 } <strong>else</strong> {<a name="326" href="#326">326</a>                     <strong>return</strong>;<a name="327" href="#327">327</a>                 }<a name="328" href="#328">328</a>             } <strong>else</strong> <strong>if</strong>(opStatus == OperationStatus.SUCCESS) {<a name="329" href="#329">329</a>                 <em class="comment">// Just inserted a brand new CrawlURI into the queue.</em><a name="330" href="#330">330</a>                 size++;<a name="331" href="#331">331</a>             }<a name="332" href="#332">332</a>     <a name="333" href="#333">333</a>             <em class="comment">// Finally, check if insert (fresh add or override) into DB was </em><a name="334" href="#334">334</a>             <em class="comment">// successful and if so check if we need to update nextReadyTime.</em><a name="335" href="#335">335</a>             <strong>if</strong>(opStatus == OperationStatus.SUCCESS){<a name="336" href="#336">336</a>                 <strong>if</strong> (curiProcessingTime &lt; nextReadyTime){<a name="337" href="#337">337</a>                     <em class="comment">// Update nextReadyTime to reflect new value.</em><a name="338" href="#338">338</a>                     setNextReadyTime(curiProcessingTime);<a name="339" href="#339">339</a>                 }<a name="340" href="#340">340</a>                 <strong>if</strong>(state == HQSTATE_EMPTY){<a name="341" href="#341">341</a>                     <em class="comment">// Definately no longer empty.</em><a name="342" href="#342">342</a>                     state = HQSTATE_READY;<a name="343" href="#343">343</a>                 }<a name="344" href="#344">344</a>             } <strong>else</strong> {<a name="345" href="#345">345</a>                 <em class="comment">// Something went wrong. Throw an exception.</em><a name="346" href="#346">346</a>                 <strong>throw</strong> <strong>new</strong> DatabaseException(<span class="string">"Error on add into database for "</span> +<a name="347" href="#347">347</a>                         <span class="string">"CrawlURI "</span> + curi.toString() + <span class="string">". "</span> + <a name="348" href="#348">348</a>                         opStatus.toString());<a name="349" href="#349">349</a>             }<a name="350" href="#350">350</a>         } <strong>catch</strong> (DatabaseException e) {<a name="351" href="#351">351</a>             <em class="comment">// Blanket catch all DBExceptions and convert to IOExceptions.</em><a name="352" href="#352">352</a>             IOException e2 = <strong>new</strong> IOException(e.getMessage());<a name="353" href="#353">353</a>             e2.setStackTrace(e.getStackTrace()); <em class="comment">//preserve original stacktrace</em><a name="354" href="#354">354</a>             <strong>throw</strong> e2; <a name="355" href="#355">355</a>         }<a name="356" href="#356">356</a>         reorder(); <em class="comment">// May need a reorder.</em><a name="357" href="#357">357</a>     }<a name="358" href="#358">358</a>     <a name="359" href="#359">359</a>     <em>/**<em>*</em></em><a name="360" href="#360">360</a> <em>     * An internal method for adding URIs to the queue. </em><a name="361" href="#361">361</a> <em>     * </em><a name="362" href="#362">362</a> <em>     * @param curi The CrawlURI to add</em><a name="363" href="#363">363</a> <em>     * @param overrideDuplicates If true then any existing CrawlURI in the DB</em><a name="364" href="#364">364</a> <em>     *                           will be overwritten. If false insert into the</em><a name="365" href="#365">365</a> <em>     *                           queue is only performed if the key doesn't </em><a name="366" href="#366">366</a> <em>     *                           already exist.</em><a name="367" href="#367">367</a> <em>     * @return The OperationStatus object returned by the put method.</em><a name="368" href="#368">368</a> <em>     * </em><a name="369" href="#369">369</a> <em>     * @throws DatabaseException</em><a name="370" href="#370">370</a> <em>     */</em><a name="371" href="#371">371</a>     <strong>protected</strong> OperationStatus strictAdd(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi,<a name="372" href="#372">372</a>         <strong>boolean</strong> overrideDuplicates)<a name="373" href="#373">373</a>     throws DatabaseException{<a name="374" href="#374">374</a>         DatabaseEntry keyEntry = <strong>new</strong> DatabaseEntry();<a name="375" href="#375">375</a>         DatabaseEntry dataEntry = <strong>new</strong> DatabaseEntry();<a name="376" href="#376">376</a>         primaryKeyBinding.objectToEntry(curi.toString(), keyEntry);<a name="377" href="#377">377</a>         crawlURIBinding.objectToEntry(curi, dataEntry);<a name="378" href="#378">378</a>         OperationStatus opStatus = <strong>null</strong>;<a name="379" href="#379">379</a>         <strong>if</strong>(overrideDuplicates){<a name="380" href="#380">380</a>             opStatus = primaryUriDB.put(<strong>null</strong>,keyEntry,dataEntry);<a name="381" href="#381">381</a>         } <strong>else</strong> {<a name="382" href="#382">382</a>             opStatus = primaryUriDB.putNoOverwrite(<strong>null</strong>,keyEntry,dataEntry);<a name="383" href="#383">383</a>         }<a name="384" href="#384">384</a>         <a name="385" href="#385">385</a>         <strong>return</strong> opStatus;<a name="386" href="#386">386</a>     }<a name="387" href="#387">387</a>     <a name="388" href="#388">388</a>     <em>/**<em>*</em></em><a name="389" href="#389">389</a> <em>     * Flush any CrawlURIs in the processingUriDB into the primaryUriDB. URIs</em><a name="390" href="#390">390</a> <em>     * flushed will have their 'time of next fetch' maintained and the </em><a name="391" href="#391">391</a> <em>     * nextReadyTime will be updated if needed.</em><a name="392" href="#392">392</a> <em>     * &lt;p></em><a name="393" href="#393">393</a> <em>     * No change is made to the list of available slots. </em><a name="394" href="#394">394</a> <em>     * </em><a name="395" href="#395">395</a> <em>     * @throws DatabaseException if one occurs while flushing</em><a name="396" href="#396">396</a> <em>     */</em><a name="397" href="#397">397</a>     <strong>protected</strong> <strong>void</strong> flushProcessingURIs() throws DatabaseException {<a name="398" href="#398">398</a>         Cursor processingCursor = processingUriDB.openCursor(<strong>null</strong>,<strong>null</strong>);<a name="399" href="#399">399</a>         DatabaseEntry keyEntry = <strong>new</strong> DatabaseEntry();<a name="400" href="#400">400</a>         DatabaseEntry dataEntry = <strong>new</strong> DatabaseEntry();<a name="401" href="#401">401</a>         <a name="402" href="#402">402</a>         <strong>while</strong>(<strong>true</strong>){<a name="403" href="#403">403</a>             OperationStatus opStatus = processingCursor.getFirst(<a name="404" href="#404">404</a>                     keyEntry, dataEntry, LockMode.DEFAULT);<a name="405" href="#405">405</a>             <a name="406" href="#406">406</a>             <strong>if</strong>(opStatus == OperationStatus.SUCCESS){<a name="407" href="#407">407</a>                 <em class="comment">// Got one!</em><a name="408" href="#408">408</a>                 <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi = <a name="409" href="#409">409</a>                     (<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a>) crawlURIBinding.entryToObject(dataEntry);<a name="410" href="#410">410</a>                 <em class="comment">// Delete it from processingUriDB</em><a name="411" href="#411">411</a>                 deleteInProcessing(curi.toString());<a name="412" href="#412">412</a>                 <em class="comment">// Add to processingUriDB;</em><a name="413" href="#413">413</a>                 strictAdd(curi,false); <em class="comment">// Ignore any duplicates. Go with the</em><a name="414" href="#414">414</a>                                        <em class="comment">// ones already in the queue.</em><a name="415" href="#415">415</a>                 <em class="comment">// Update nextReadyTime if needed.</em><a name="416" href="#416">416</a>                 <strong>long</strong> curiNextReadyTime = curi.getLong(<a name="417" href="#417">417</a>                         A_TIME_OF_NEXT_PROCESSING);<a name="418" href="#418">418</a>                 <strong>if</strong>(curiNextReadyTime&lt;nextReadyTime){<a name="419" href="#419">419</a>                     setNextReadyTime(curiNextReadyTime);<a name="420" href="#420">420</a>                 }<a name="421" href="#421">421</a>             } <strong>else</strong> {<a name="422" href="#422">422</a>                 <em class="comment">// No more entries in processingUriDB</em><a name="423" href="#423">423</a>                 processingCursor.close();