<a name="121" href="#121">121</a> <strong>import</strong> com.sleepycat.bind.tuple.StringBinding;<a name="122" href="#122">122</a> <strong>import</strong> com.sleepycat.collections.StoredSortedMap;<a name="123" href="#123">123</a> <strong>import</strong> com.sleepycat.je.Database;<a name="124" href="#124">124</a> <strong>import</strong> com.sleepycat.je.DatabaseConfig;<a name="125" href="#125">125</a> <strong>import</strong> com.sleepycat.je.DatabaseException;<a name="126" href="#126">126</a> <strong>import</strong> com.sleepycat.je.Environment;<a name="127" href="#127">127</a> <a name="128" href="#128">128</a> <em>/**<em>*</em></em><a name="129" href="#129">129</a> <em> * HTTP fetcher that uses &lt;a</em><a name="130" href="#130">130</a> <em> * href="<a href="http://jakarta.apache.org/commons/httpclient/" target="alexandria_uri">http://jakarta.apache.org/commons/httpclient/</a>">Apache Jakarta Commons</em><a name="131" href="#131">131</a> <em> * HttpClient&lt;/a> library.</em><a name="132" href="#132">132</a> <em> *</em><a name="133" href="#133">133</a> <em> * @author Gordon Mohr</em><a name="134" href="#134">134</a> <em> * @author Igor Ranitovic</em><a name="135" href="#135">135</a> <em> * @author others</em><a name="136" href="#136">136</a> <em> * @version $Id: FetchHTTP.java 5093 2007-04-24 21:48:34Z gojomo $</em><a name="137" href="#137">137</a> <em> */</em><a name="138" href="#138">138</a> <strong>public</strong> <strong>class</strong> <a href="../../../../org/archive/crawler/fetcher/FetchHTTP.html">FetchHTTP</a> <strong>extends</strong> <a href="../../../../org/archive/crawler/framework/Processor.html">Processor</a><a name="139" href="#139">139</a> implements CoreAttributeConstants, FetchStatusCodes, <a href="../../../../org/archive/crawler/event/CrawlStatusListener.html">CrawlStatusListener</a> {<a name="140" href="#140">140</a>     <em class="comment">// be robust against trivial implementation changes</em><a name="141" href="#141">141</a>     <strong>private</strong> <strong>static</strong> <strong>final</strong> <strong>long</strong> serialVersionUID =<a name="142" href="#142">142</a>         ArchiveUtils.<strong>class</strong>nameBasedUID(FetchHTTP.<strong>class</strong>,1);<a name="143" href="#143">143</a>     <a name="144" href="#144">144</a>     <strong>private</strong> <strong>static</strong> Logger logger = Logger.getLogger(FetchHTTP.<strong>class</strong>.getName());<a name="145" href="#145">145</a> <a name="146" href="#146">146</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_HTTP_PROXY_HOST = A_HTTP_PROXY_HOST;<a name="147" href="#147">147</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_HTTP_PROXY_PORT = A_HTTP_PROXY_PORT;<a name="148" href="#148">148</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_TIMEOUT_SECONDS = <span class="string">"timeout-seconds"</span>;<a name="149" href="#149">149</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_SOTIMEOUT_MS = <span class="string">"sotimeout-ms"</span>;<a name="150" href="#150">150</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_MAX_LENGTH_BYTES = <span class="string">"max-length-bytes"</span>;<a name="151" href="#151">151</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_LOAD_COOKIES = <span class="string">"load-cookies-from-file"</span>;<a name="152" href="#152">152</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_SAVE_COOKIES = <span class="string">"save-cookies-to-file"</span>;<a name="153" href="#153">153</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_ACCEPT_HEADERS = <span class="string">"accept-headers"</span>;<a name="154" href="#154">154</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_DEFAULT_ENCODING = <span class="string">"default-encoding"</span>;<a name="155" href="#155">155</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_DIGEST_CONTENT = <span class="string">"digest-content"</span>;<a name="156" href="#156">156</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_DIGEST_ALGORITHM = <span class="string">"digest-algorithm"</span>;<a name="157" href="#157">157</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_FETCH_BANDWIDTH_MAX = <span class="string">"fetch-bandwidth"</span>;<a name="158" href="#158">158</a>    <a name="159" href="#159">159</a>     <em>/**<em>*</em></em><a name="160" href="#160">160</a> <em>     * SSL trust level setting attribute name.</em><a name="161" href="#161">161</a> <em>     */</em><a name="162" href="#162">162</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_TRUST = <span class="string">"trust-level"</span>;<a name="163" href="#163">163</a>     <a name="164" href="#164">164</a>     <strong>private</strong> <strong>static</strong> Integer DEFAULT_TIMEOUT_SECONDS = <strong>new</strong> Integer(1200);<a name="165" href="#165">165</a>     <strong>private</strong> <strong>static</strong> Integer DEFAULT_SOTIMEOUT_MS = <strong>new</strong> Integer(20000);<a name="166" href="#166">166</a>     <strong>private</strong> <strong>static</strong> Long DEFAULT_MAX_LENGTH_BYTES = <strong>new</strong> Long(0);<a name="167" href="#167">167</a>     <strong>private</strong> <strong>static</strong> Integer DEFAULT_FETCH_BANDWIDTH_MAX = 0;<a name="168" href="#168">168</a> <a name="169" href="#169">169</a>     <em>/**<em>*</em></em><a name="170" href="#170">170</a> <em>     * This is the default value pre-1.4. Needs special handling else</em><a name="171" href="#171">171</a> <em>     * treated as negative number doing math later in processing.</em><a name="172" href="#172">172</a> <em>     */</em><a name="173" href="#173">173</a>     <strong>private</strong> <strong>static</strong> <strong>long</strong> OLD_DEFAULT_MAX_LENGTH_BYTES = 9223372036854775807L;<a name="174" href="#174">174</a> <a name="175" href="#175">175</a>     <em>/**<em>*</em></em><a name="176" href="#176">176</a> <em>     * Default character encoding to use for pages that do not specify.</em><a name="177" href="#177">177</a> <em>     */</em><a name="178" href="#178">178</a>     <strong>private</strong> <strong>static</strong> String DEFAULT_CONTENT_CHARSET = Heritrix.DEFAULT_ENCODING;<a name="179" href="#179">179</a> <a name="180" href="#180">180</a>     <em>/**<em>*</em></em><a name="181" href="#181">181</a> <em>     * Default whether to perform on-the-fly digest hashing of content-bodies.</em><a name="182" href="#182">182</a> <em>     */</em><a name="183" href="#183">183</a>     <strong>static</strong> Boolean DEFAULT_DIGEST_CONTENT = <strong>new</strong> Boolean(<strong>true</strong>);<a name="184" href="#184">184</a>           <a name="185" href="#185">185</a>     <em>/**<em>*</em></em><a name="186" href="#186">186</a> <em>     * The different digest algorithms to choose between, </em><a name="187" href="#187">187</a> <em>     * SHA-1 or MD-5 at the moment. </em><a name="188" href="#188">188</a> <em>     */</em><a name="189" href="#189">189</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String SHA1 = <span class="string">"sha1"</span>;<a name="190" href="#190">190</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String MD5 = <span class="string">"md5"</span>;<a name="191" href="#191">191</a>     <strong>public</strong> <strong>static</strong> String [] DIGEST_ALGORITHMS = {SHA1, MD5};<a name="192" href="#192">192</a>    <a name="193" href="#193">193</a>     <em>/**<em>*</em></em><a name="194" href="#194">194</a> <em>     * Default algorithm to use for message disgesting.</em><a name="195" href="#195">195</a> <em>     */</em><a name="196" href="#196">196</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String  DEFAULT_DIGEST_ALGORITHM = SHA1; <a name="197" href="#197">197</a>     <a name="198" href="#198">198</a>     <strong>private</strong> <strong>transient</strong> HttpClient http = <strong>null</strong>;<a name="199" href="#199">199</a> <a name="200" href="#200">200</a>     <em>/**<em>*</em></em><a name="201" href="#201">201</a> <em>     * How many 'instant retries' of HttpRecoverableExceptions have occurred</em><a name="202" href="#202">202</a> <em>     * </em><a name="203" href="#203">203</a> <em>     * Would like it to be 'long', but longs aren't atomic</em><a name="204" href="#204">204</a> <em>     */</em><a name="205" href="#205">205</a>     <strong>private</strong> <strong>int</strong> recoveryRetries = 0;<a name="206" href="#206">206</a> <a name="207" href="#207">207</a>     <em>/**<em>*</em></em><a name="208" href="#208">208</a> <em>     * Count of crawl uris handled.</em><a name="209" href="#209">209</a> <em>     * Would like to be 'long', but longs aren't atomic</em><a name="210" href="#210">210</a> <em>     */</em><a name="211" href="#211">211</a>     <strong>private</strong> <strong>int</strong> curisHandled = 0;<a name="212" href="#212">212</a>         <a name="213" href="#213">213</a>     <em>/**<em>*</em></em><a name="214" href="#214">214</a> <em>     * Rules to apply mid-fetch, just after receipt of the response</em><a name="215" href="#215">215</a> <em>     * headers before we start to download body.</em><a name="216" href="#216">216</a> <em>     */</em><a name="217" href="#217">217</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_MIDFETCH_DECIDE_RULES = <span class="string">"midfetch-decide-rules"</span>;<a name="218" href="#218">218</a>     <a name="219" href="#219">219</a>     <em>/**<em>*</em></em><a name="220" href="#220">220</a> <em>     * What to log if midfetch abort.</em><a name="221" href="#221">221</a> <em>     */</em><a name="222" href="#222">222</a>     <strong>private</strong> <strong>static</strong> <strong>final</strong> String MIDFETCH_ABORT_LOG = <span class="string">"midFetchAbort"</span>;<a name="223" href="#223">223</a>     <a name="224" href="#224">224</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_SEND_CONNECTION_CLOSE =<a name="225" href="#225">225</a>         <span class="string">"send-connection-close"</span>;<a name="226" href="#226">226</a>     <strong>private</strong> <strong>static</strong> <strong>final</strong> Header HEADER_SEND_CONNECTION_CLOSE =<a name="227" href="#227">227</a>         <strong>new</strong> Header(<span class="string">"Connection"</span>, <span class="string">"close"</span>);<a name="228" href="#228">228</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_SEND_REFERER = <span class="string">"send-referer"</span>;<a name="229" href="#229">229</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_SEND_RANGE = <span class="string">"send-range"</span>;<a name="230" href="#230">230</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_SEND_IF_MODIFIED_SINCE = <span class="string">"send-if-modified-since"</span>;<a name="231" href="#231">231</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_SEND_IF_NONE_MATCH = <span class="string">"send-if-none-match"</span>;<a name="232" href="#232">232</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String REFERER = <span class="string">"Referer"</span>;<a name="233" href="#233">233</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String RANGE = <span class="string">"Range"</span>;<a name="234" href="#234">234</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String RANGE_PREFIX = <span class="string">"bytes=0-"</span>;<a name="235" href="#235">235</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String HTTP_SCHEME = <span class="string">"http"</span>;<a name="236" href="#236">236</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String HTTPS_SCHEME = <span class="string">"https"</span>;<a name="237" href="#237">237</a>     <a name="238" href="#238">238</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_IGNORE_COOKIES = <span class="string">"ignore-cookies"</span>;<a name="239" href="#239">239</a>     <strong>private</strong> <strong>static</strong> Boolean DEFAULT_IGNORE_COOKIES = <strong>new</strong> Boolean(false);<a name="240" href="#240">240</a> <a name="241" href="#241">241</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_BDB_COOKIES = <span class="string">"use-bdb-for-cookies"</span>;<a name="242" href="#242">242</a>     <strong>private</strong> <strong>static</strong> Boolean DEFAULT_BDB_COOKIES = <strong>new</strong> Boolean(<strong>true</strong>);<a name="243" href="#243">243</a>     <a name="244" href="#244">244</a>     <strong>public</strong> <strong>static</strong> <strong>final</strong> String ATTR_LOCAL_ADDRESS = <span class="string">"bind-address"</span>;<a name="245" href="#245">245</a>     <a name="246" href="#246">246</a>     <em>/**<em>*</em></em><a name="247" href="#247">247</a> <em>     * Database backing cookie map, if using BDB</em><a name="248" href="#248">248</a> <em>     */</em><a name="249" href="#249">249</a>     <strong>protected</strong> Database cookieDb;