<a name="556" href="#556">556</a>             }<a name="557" href="#557">557</a>         }<a name="558" href="#558">558</a>         <em class="comment">// save ignored items (if any) where they can be consulted later</em><a name="559" href="#559">559</a>         saveIgnoredItems(ignoredWriter.toString(), controller.getDisk());<a name="560" href="#560">560</a>         logger.info(<span class="string">"finished"</span>);<a name="561" href="#561">561</a>     }<a name="562" href="#562">562</a> <a name="563" href="#563">563</a>     <em>/**<em>*</em></em><a name="564" href="#564">564</a> <em>     * Dump ignored seed items (if any) to disk; delete file otherwise.</em><a name="565" href="#565">565</a> <em>     * Static to allow non-derived sibling classes (frontiers not yet </em><a name="566" href="#566">566</a> <em>     * subclassed here) to reuse.</em><a name="567" href="#567">567</a> <em>     * </em><a name="568" href="#568">568</a> <em>     * @param ignoredItems</em><a name="569" href="#569">569</a> <em>     * @param dir </em><a name="570" href="#570">570</a> <em>     */</em><a name="571" href="#571">571</a>     <strong>public</strong> <strong>static</strong> <strong>void</strong> saveIgnoredItems(String ignoredItems, File dir) {<a name="572" href="#572">572</a>         File ignoredFile = <strong>new</strong> File(dir, IGNORED_SEEDS_FILENAME);<a name="573" href="#573">573</a>         <strong>if</strong>(ignoredItems==<strong>null</strong> | ignoredItems.length()>0) {<a name="574" href="#574">574</a>             <strong>try</strong> {<a name="575" href="#575">575</a>                 BufferedWriter bw = <strong>new</strong> BufferedWriter(<strong>new</strong> FileWriter(ignoredFile));<a name="576" href="#576">576</a>                 bw.write(ignoredItems);<a name="577" href="#577">577</a>                 bw.close();<a name="578" href="#578">578</a>             } <strong>catch</strong> (IOException e) {<a name="579" href="#579">579</a>                 <em class="comment">// TODO make an alert?</em><a name="580" href="#580">580</a>                 e.printStackTrace();<a name="581" href="#581">581</a>             }<a name="582" href="#582">582</a>         } <strong>else</strong> {<a name="583" href="#583">583</a>             <em class="comment">// delete any older file (if any)</em><a name="584" href="#584">584</a>             ignoredFile.delete();<a name="585" href="#585">585</a>         }<a name="586" href="#586">586</a>     }<a name="587" href="#587">587</a> <a name="588" href="#588">588</a>     <strong>protected</strong> <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> asCrawlUri(<a href="../../../../org/archive/crawler/datamodel/CandidateURI.html">CandidateURI</a> caUri) {<a name="589" href="#589">589</a>         <a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi;<a name="590" href="#590">590</a>         <strong>if</strong> (caUri instanceof CrawlURI) {<a name="591" href="#591">591</a>             curi = (CrawlURI)caUri;<a name="592" href="#592">592</a>         } <strong>else</strong> {<a name="593" href="#593">593</a>             curi = CrawlURI.from(caUri, nextOrdinal.getAndIncrement());<a name="594" href="#594">594</a>         }<a name="595" href="#595">595</a>         curi.setClassKey(getClassKey(curi));<a name="596" href="#596">596</a>         <strong>return</strong> curi;<a name="597" href="#597">597</a>     }<a name="598" href="#598">598</a> <a name="599" href="#599">599</a>     <em>/**<em>*</em></em><a name="600" href="#600">600</a> <em>     * @param now</em><a name="601" href="#601">601</a> <em>     * @throws InterruptedException</em><a name="602" href="#602">602</a> <em>     * @throws EndedException</em><a name="603" href="#603">603</a> <em>     */</em><a name="604" href="#604">604</a>     <strong>protected</strong> <strong>synchronized</strong> <strong>void</strong> preNext(<strong>long</strong> now) throws InterruptedException,<a name="605" href="#605">605</a>             <a href="../../../../org/archive/crawler/framework/exceptions/EndedException.html">EndedException</a> {<a name="606" href="#606">606</a>         <strong>if</strong> (<strong>this</strong>.controller == <strong>null</strong>) {<a name="607" href="#607">607</a>             <strong>return</strong>;<a name="608" href="#608">608</a>         }<a name="609" href="#609">609</a>         <a name="610" href="#610">610</a>         <em class="comment">// Check completion conditions</em><a name="611" href="#611">611</a>         <strong>if</strong> (<strong>this</strong>.controller.atFinish()) {<a name="612" href="#612">612</a>             <strong>if</strong> (((Boolean)getUncheckedAttribute(<strong>null</strong>, ATTR_PAUSE_AT_FINISH))<a name="613" href="#613">613</a>                     .booleanValue()) {<a name="614" href="#614">614</a>                 <strong>this</strong>.controller.requestCrawlPause();<a name="615" href="#615">615</a>             } <strong>else</strong> {<a name="616" href="#616">616</a>                 <strong>this</strong>.controller.beginCrawlStop();<a name="617" href="#617">617</a>             }<a name="618" href="#618">618</a>         }<a name="619" href="#619">619</a> <a name="620" href="#620">620</a>         <em class="comment">// enforce operator pause</em><a name="621" href="#621">621</a>         <strong>if</strong> (shouldPause) {<a name="622" href="#622">622</a>             <strong>while</strong> (shouldPause) {<a name="623" href="#623">623</a>                 <strong>this</strong>.controller.toePaused();<a name="624" href="#624">624</a>                 wait();<a name="625" href="#625">625</a>             }<a name="626" href="#626">626</a>             <em class="comment">// exitted pause; possibly finish regardless of pause-at-finish</em><a name="627" href="#627">627</a>             <strong>if</strong> (controller != <strong>null</strong> &amp;&amp; controller.atFinish()) {<a name="628" href="#628">628</a>                 <strong>this</strong>.controller.beginCrawlStop();<a name="629" href="#629">629</a>             }<a name="630" href="#630">630</a>         }<a name="631" href="#631">631</a> <a name="632" href="#632">632</a>         <em class="comment">// enforce operator terminate or thread retirement</em><a name="633" href="#633">633</a>         <strong>if</strong> (shouldTerminate<a name="634" href="#634">634</a>                 || ((ToeThread)Thread.currentThread()).shouldRetire()) {<a name="635" href="#635">635</a>             <strong>throw</strong> <strong>new</strong> <a href="../../../../org/archive/crawler/framework/exceptions/EndedException.html">EndedException</a>(<span class="string">"terminated"</span>);<a name="636" href="#636">636</a>         }<a name="637" href="#637">637</a> <a name="638" href="#638">638</a>         enforceBandwidthThrottle(now);<a name="639" href="#639">639</a>     }<a name="640" href="#640">640</a> <a name="641" href="#641">641</a>     <em>/**<em>*</em></em><a name="642" href="#642">642</a> <em>     * Perform any special handling of the CrawlURI, such as promoting its URI</em><a name="643" href="#643">643</a> <em>     * to seed-status, or preferencing it because it is an embed.</em><a name="644" href="#644">644</a> <em>     * </em><a name="645" href="#645">645</a> <em>     * @param curi</em><a name="646" href="#646">646</a> <em>     */</em><a name="647" href="#647">647</a>     <strong>protected</strong> <strong>void</strong> applySpecialHandling(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi) {<a name="648" href="#648">648</a>         <strong>if</strong> (curi.isSeed() &amp;&amp; curi.getVia() != <strong>null</strong><a name="649" href="#649">649</a>                 &amp;&amp; curi.flattenVia().length() > 0) {<a name="650" href="#650">650</a>             <em class="comment">// The only way a seed can have a non-empty via is if it is the</em><a name="651" href="#651">651</a>             <em class="comment">// result of a seed redirect. Add it to the seeds list.</em><a name="652" href="#652">652</a>             <em class="comment">//</em><a name="653" href="#653">653</a>             <em class="comment">// This is a feature. This is handling for case where a seed</em><a name="654" href="#654">654</a>             <em class="comment">// gets immediately redirected to another page. What we're doing is</em><a name="655" href="#655">655</a>             <em class="comment">// treating the immediate redirect target as a seed.</em><a name="656" href="#656">656</a>             <strong>this</strong>.controller.getScope().addSeed(curi);<a name="657" href="#657">657</a>             <em class="comment">// And it needs rapid scheduling.</em><a name="658" href="#658">658</a> 	    <strong>if</strong> (curi.getSchedulingDirective() == CandidateURI.NORMAL)<a name="659" href="#659">659</a>                 curi.setSchedulingDirective(CandidateURI.MEDIUM);<a name="660" href="#660">660</a>         }<a name="661" href="#661">661</a> <a name="662" href="#662">662</a>         <em class="comment">// optionally preferencing embeds up to MEDIUM</em><a name="663" href="#663">663</a>         <strong>int</strong> prefHops = ((Integer)getUncheckedAttribute(curi,<a name="664" href="#664">664</a>                 ATTR_PREFERENCE_EMBED_HOPS)).intValue();<a name="665" href="#665">665</a>         <strong>if</strong> (prefHops > 0) {<a name="666" href="#666">666</a>             <strong>int</strong> embedHops = curi.getTransHops();<a name="667" href="#667">667</a>             <strong>if</strong> (embedHops > 0 &amp;&amp; embedHops &lt;= prefHops<a name="668" href="#668">668</a>                     &amp;&amp; curi.getSchedulingDirective() == CandidateURI.NORMAL) {<a name="669" href="#669">669</a>                 <em class="comment">// number of embed hops falls within the preferenced range, and</em><a name="670" href="#670">670</a>                 <em class="comment">// uri is not already MEDIUM -- so promote it</em><a name="671" href="#671">671</a>                 curi.setSchedulingDirective(CandidateURI.MEDIUM);<a name="672" href="#672">672</a>             }<a name="673" href="#673">673</a>         }<a name="674" href="#674">674</a>     }<a name="675" href="#675">675</a> <a name="676" href="#676">676</a>     <em>/**<em>*</em></em><a name="677" href="#677">677</a> <em>     * Perform fixups on a CrawlURI about to be returned via next().</em><a name="678" href="#678">678</a> <em>     * </em><a name="679" href="#679">679</a> <em>     * @param curi</em><a name="680" href="#680">680</a> <em>     *            CrawlURI about to be returned by next()</em><a name="681" href="#681">681</a> <em>     * @param q</em><a name="682" href="#682">682</a> <em>     *            the queue from which the CrawlURI came</em><a name="683" href="#683">683</a> <em>     */</em><a name="684" href="#684">684</a>     <strong>protected</strong> <strong>void</strong> noteAboutToEmit(<a href="../../../../org/archive/crawler/datamodel/CrawlURI.html">CrawlURI</a> curi, <a href="../../../../org/archive/crawler/frontier/WorkQueue.html">WorkQueue</a> q) {<a name="685" href="#685">685</a>         curi.setHolder(q);<a name="686" href="#686">686</a>         <em class="comment">// if (curi.getServer() == null) {</em><a name="687" href="#687">687</a>         <em class="comment">//    // TODO: perhaps short-circuit the emit here,</em><a name="688" href="#688">688</a>         <em class="comment">//    // because URI will be rejected as unfetchable</em><a name="689" href="#689">689</a>         <em class="comment">// }</em><a name="690" href="#690">690</a>         doJournalEmitted(curi);<a name="691" href="#691">691</a>     }<a name="692" href="#692">692</a> <a name="693" href="#693">693</a>     <em>/**<em>*</em></em><a name="694" href="#694">694</a> <em>     * @param curi</em><a name="695" href="#695">695</a> <em>     * @return the CrawlServer to be associated with this CrawlURI</em><