Newer
Older
Digital_Repository / Repositories / statistics / scripts / eprints-usage_src.php
  1. <?php
  2.  
  3. /* NJS 2007-07-24
  4. The database structure changed between versions 2.x and 3.x of
  5. EPrints, so we now need to check the major version number and alter
  6. the queries appropriately. Use only the MAJOR version number (i.e.,
  7. 2 or 3, don't include the release number).
  8. */
  9. $eprints_version = ##EPRINTS_VERSION##;
  10.  
  11. /* NJS 2006-04-28
  12. In earlier versions of this script, which eprints to count was
  13. determined by comparing the request date of the eprint against the
  14. "lastproc" date of this script (i.e., minimum time unit one day).
  15. This was fine if you only ran the script once per day, but if you ran
  16. it more than that, it counted multiple times requests whose
  17. $request_date == $lastproc. For example, if you ran this script five
  18. times per day, all the downloads that occurred during that day would
  19. be counted EVERY TIME this script ran, thus overinflating your stats
  20. by a factor of up to five :(
  21. The solution is to use the full time stamp for comparison rather than
  22. just the date. This timestamp MUST include time zone information so
  23. that things don't get screwed up by daylight saving time. As long as
  24. this is done consistently, there's no need to do things like convert
  25. to GMT, for example.
  26. The very first thing we need to do is grab the current time stamp
  27. with time zone, which will later be stored in the database as the
  28. "lastproc" time. This needs to happen first so that we don't "lose"
  29. any requests that occur while the script is running.
  30. */
  31. $start_time = date('Y-m-d H:i:s O');
  32.  
  33. /* NJS 2007-01-30
  34. A further twist! The original script ignored log lines that had a
  35. date falling before $lastproc, i.e., if log line date < $lastproc
  36. then it's already been dealt with. This is all fine. However, it
  37. didn't bother checking for log lines that were written after the
  38. script started running (i.e. log line date >= $start_time).
  39. Why is this a problem? We're reading the live Apache log file, so
  40. it's quite likely that new lines will be written to it after the
  41. script has started (i.e., after $start_time). Suppose $start_time is
  42. '2006-06-15 14:03:15 +1200', $lastproc is '2006-06-15 12:03:15 +1200'
  43. (i.e., the script is run every two hours) and the log file contains
  44. lines with the following dates:
  45. '2006-06-15 10:03:15 +1200' [1] <-- written before $lastproc
  46. '2006-06-15 12:03:14 +1200' [2] <-- written before $lastproc
  47. '2006-06-15 13:03:15 +1200' [3] <-- written before $start_time
  48. '2006-06-15 14:03:14 +1200' [4] <-- written before $start_time
  49. '2006-06-15 14:03:15 +1200' [5] <-- written at $start_time
  50. '2006-06-15 14:03:16 +1200' [6] <-- written after $start_time
  51.  
  52. During this run, dates [1] and [2] are both < $lastproc and thus
  53. ignored. The remaining four dates ([4]--[6]) are >= $lastproc and
  54. thus processed.
  55.  
  56. Two hours later, the script runs again, this time with $start_time
  57. set to '2006-06-15 16:03:15 +1200' and $lastproc to '2006-06-15
  58. 14:03:15 +1200'. Dates [1] through [4] are all < $lastproc and
  59. thus ignored. However, dates [5] and [6] are both >= $lastproc
  60. and are processed a second time, resulting in a duplicate entry
  61. in the database.
  62. The solution is to ignore any log line entries that occur at or after
  63. (>=) $start_time. In the example above, this would mean that in the
  64. first run, dates [1], [2], [5] and [6] would be ignored and dates [3]
  65. and [4] processed. In the second run, dates [1]--[4] would be ignored
  66. and dates [5] and [6] processed.
  67. */
  68. $test_starttime = strtotime($start_time);
  69.  
  70.  
  71. // NJS 2005-12-09 Switched to GeoIP from GeoIP:IPfree.
  72. include("geoip.inc");
  73.  
  74. $gi = geoip_open("##GEOIP_DATABASE##",GEOIP_STANDARD);
  75.  
  76. /*
  77.  
  78. Apache log for ePrints uses this format:
  79. LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
  80.  
  81. If the log format differs the regular expression matching would need to be adjusted.
  82. Parse:
  83. ip
  84. date YYYY MM DD
  85. archive ID
  86.  
  87. */
  88.  
  89. // Web server log files
  90. $log_dir = '##APACHE_LOG_LOCATION##';
  91. $log_file = array(
  92. 'otago_eprints' => '##APACHE_LOG_NAME_1##',
  93. 'cardrona' => '##APACHE_LOG_NAME_2##',
  94. );
  95.  
  96.  
  97. // eprintstats db
  98. $sqlserver = 'localhost';
  99. $sqluser = 'eprintstatspriv';
  100. $sqlpass = 'AuldGrizzel';
  101. $sqldatabase = 'eprintstats';
  102.  
  103. /* NJS 2006-05-26
  104. SQL details of your ePrints installation(s). This has now been
  105. generalised to work with multiple archives. For each archive that you
  106. have, add an entry to this array in the following format:
  107.  
  108. 'archive_name' => array(
  109. 'sqlserver' => 'db_host',
  110. 'dbname' => 'database_name',
  111. 'username' => 'user_name',
  112. 'password' => 'password',
  113. ),
  114.  
  115. NJS 2007-07-16: Added support for different database name.
  116. Usually, archive_name, database_name and user_name are the same, but they
  117. don't have to be.
  118. */
  119. $eprintsdbs = array(
  120. 'otago_eprints' => array(
  121. 'sqlserver' => 'localhost',
  122. 'dbname' => 'otago_eprints_v3',
  123. 'username' => 'otago_eprints',
  124. 'password' => 'DrSyntaxRidesAgain',
  125. ),
  126. 'cardrona' => array(
  127. 'sqlserver' => 'localhost',
  128. 'dbname' => 'cardrona_v3',
  129. 'username' => 'cardrona',
  130. 'password' => 'DrSyntaxRidesAgain',
  131. ),
  132. );
  133.  
  134. /* NJS 2005-12-16
  135. IP address ranges for your local Intranet(s). You can have multiple
  136. ranges of IP addresses, each with a different "country name", so that
  137. they will appear as separate entries in the by country stats pages.
  138. You should use a different country code for each range (ISO 3166-1
  139. specifies the range XA through XZ as "user-assignable", so you can use
  140. codes from there as necessary), and create flag icons as appropriate.
  141.  
  142. Each address range key is the name that will appear in the statistics
  143. database (the "country name"), followed by a comma, followed by the
  144. appropriate ISO 3166-1 country code as noted above. Each entry in the
  145. range is either a single IP address, or an array specifying a lower and
  146. upper bound for a contiguous IP address range (see example below).
  147.  
  148. All IP addresses must be converted to long values using the ip2long()
  149. function before being stored.
  150.  
  151. Note that address ranges may overlap. The script will use the first
  152. range that matches a given IP, so list the ranges in the correct order
  153. of precedence for your needs.
  154.  
  155. Example:
  156.  
  157. $local_IPs = array(
  158. 'Repository Admin,XA' => array(
  159. ip2long('192.168.1.5'),
  160. ip2long('192.168.1.22'),
  161. array(
  162. lower => ip2long('192.168.1.30'),
  163. upper => ip2long('192.168.1.35'),
  164. ),
  165. ),
  166. 'Our Intranet,XI' => array(
  167. array(
  168. lower => ip2long('192.168.1.0'),
  169. upper => ip2long('192.168.255.255'),
  170. ),
  171. ),
  172. );
  173.  
  174. 'Repository Admin' covers the IP addresses 192.168.1.5, 192.168.1.22 and
  175. the range 192.168.1.30 to 192.168.1.35, inclusive. 'Our Intranet' covers
  176. the range 192.168.1.0 to 192.168.255.255, inclusive. A machine will only
  177. match the 'Our Intranet' range if it first fails to match the
  178. 'Repository Admin' range.
  179. */
  180. $local_IPs = array(
  181. 'Repository Admin,XA' => array(
  182. ip2long('139.80.75.110'), // Nigel @ Uni
  183. ip2long('60.234.209.74'), // Nigel @ home
  184. ip2long('139.80.92.138'), // Monica & Jeremy
  185. ip2long('139.80.92.151'), // @ Uni
  186. ip2long('203.89.162.155'), // Monica @ home
  187. ip2long('139.80.81.50'), // eprints.otago.ac.nz
  188. ip2long('172.20.1.50'), // eprints.otago.ac.nz pre-switch
  189. ip2long('172.20.1.1'), // eprints.otago.ac.nz pre-switch
  190. ),
  191. 'Otago Intranet,XI' => array(
  192. array(
  193. 'lower' => ip2long('139.80.0.0'),
  194. 'upper' => ip2long('139.80.127.255'),
  195. ),
  196. ),
  197. );
  198.  
  199. /* NJS 2007-01-26
  200. Patterns to match various search engine bots. Ideally, we'd use a similar
  201. mechanism to the $local_IPs variable above, but this isn't feasible because
  202. we'd need to know the IP ranges for the likes of Google, for example. This
  203. clearly isn't possible in practice.
  204.  
  205. Fortunately, most search bots insert a readily identifiable string into
  206. the user-agent part of the HTTP response, which gets recorded in the Apache
  207. log file. We can look for these and re-code log entries as appropriate.
  208.  
  209. The format of this list is similar to that of the $local_IPs variable.
  210. The key is the "country name" (in this case the name of the search
  211. engine) plus a non-standard four-character country code starting with
  212. "X@", separated by a comma. Each key value has an associated list of
  213. corresponding regular expressions that can occur in the user-agent part
  214. of the Apache log entry. If any one of these REs matches the user-agent
  215. part of the log entry, then we should re-code the country appropriately.
  216.  
  217. A four-character code is used because that what the database allows, and
  218. it avoids having to reserve several of the "X" country codes for search
  219. engines.
  220. */
  221. $bot_patterns = array(
  222. // Yahoo! (http://www.yahoo.com/)
  223. 'Yahoo!,X@YH' => array(
  224. '/yahoo! slurp/i',
  225. '/yahooseeker/i',
  226. ),
  227. // Windows Live Search (http://search.msn.com/)
  228. 'Windows Live Search,X@MS' => array(
  229. '/msnbot/i',
  230. ),
  231. // Google (http://www.google.com/)
  232. 'Google,X@GG' => array(
  233. '/googlebot/i',
  234. ),
  235. // Ask.com (http://www.ask.com/)
  236. 'Ask.com,X@AC' => array(
  237. '/ask jeeves\/teoma/i',
  238. ),
  239. // Everything else I could find in our log files :)
  240. 'Other search engine,X@OS' => array(
  241. // TAMU Internet Research Lab (http://irl.cs.tamu.edu/)
  242. '/http:\/\/irl\.cs\.tamu\.edu\/crawler/i',
  243. // Alexa web search (http://www.alexa.com/)
  244. '/ia_archiver/i',
  245. // TrueKnowledge for Web (http://www.authoritativeweb.com/)
  246. '/converacrawler/i',
  247. // Majestic 12 distributed search engine (http://www.majestic12.co.uk/)
  248. '/mj12bot/i',
  249. // Picsearch (http://www.picsearch.com/)
  250. '/psbot/i',
  251. // Exalead (http://www.exalead.com/search)
  252. '/exabot/i',
  253. // Cazoodle (note cazoodle.com doesn't exist)
  254. '/cazoodlebot crawler/i',
  255. '/mqbot@cazoodle\.com/i',
  256. // Gigablast (http://www.gigablast.com/)
  257. '/gigabot/i',
  258. // Houxou (http://www.houxou.com/)
  259. '/houxoucrawler/i',
  260. '/crawler at houxou dot com/i',
  261. // IBM Almaden Research Center Computer Science group (http://www.almaden.ibm.com/cs/)
  262. '/http:\/\/www\.almaden\.ibm\.com\/cs\/crawler/i',
  263. // Goo? (http://help.goo.ne.jp/)
  264. '/ichiro/i',
  265. // Daum Communications Corp (Korea)
  266. '/edacious & intelligent web robot/i',
  267. '/daum communications corp/i',
  268. '/daum web robot/i',
  269. '/msie is not me/i',
  270. '/daumoa/i',
  271. // Girafa (http://www.girafa.com/)
  272. '/girafabot/i',
  273. // The Generations Network (http://www.myfamilyinc.com/)
  274. '/myfamilybot/i',
  275. // Naver? (http://www.naver.com/)
  276. '/naverbot/i',
  277. // WiseNut (http://www.wisenutbot.com/)
  278. '/zyborg/i',
  279. '/wn-[0-9]+\.zyborg@looksmart\.net/i',
  280. // Accelobot (http://www.accelobot.com/)
  281. // This one seems particularly busy!
  282. '/heritrix/i',
  283. // Seeqpod (http://www.seeqpod.com/)
  284. '/seeqpod-vertical-crawler/i',
  285. // University of Illinois at Urbana-Champaign, Computer Science (http://www.cs.uiuc.edu/)
  286. '/mqbot crawler/i',
  287. '/mqbot@cs\.uiuc\.edu/i',
  288. // Microsoft Research (http://research.microsoft.com/)
  289. '/msrbot/i',
  290. // Nusearch
  291. '/nusearch spider/i',
  292. // SourceForge (http://www.sf.net/)
  293. '/nutch-agent@lists\.sourceforge\.net/i',
  294. // Lucene (http://lucene.apache.org/)
  295. '/nutch-agent@lucene\.apache\.org/i',
  296. '/raphael@unterreuth.de/i',
  297. // Computer Science, University of Washington (http://cs.washington.edu/)
  298. '/nutch running at uw/i',
  299. '/sycrawl@cs\.washington\.edu/i',
  300. // Chikayama & Taura Laboratory, University of Tokyo (http://www.logos.ic.i.u-tokyo.ac.jp/)
  301. '/shim-crawler/i',
  302. '/crawl@logos\.ic\.i\.u-tokyo\.ac\.jp/i',
  303. // Sproose (http://www.sproose.com/)
  304. '/sproose bot/i',
  305. '/crawler@sproose\.com/i',
  306. // Turnitin (http://www.turnitin.com/)
  307. '/turnitinbot/i',
  308. // WISH Project (http://wish.slis.tsukuba.ac.jp/)
  309. '/wish-project/i',
  310. // WWWster
  311. '/wwwster/i',
  312. '/gue@cis\.uni-muenchen\.de/i',
  313. // Forex Trading Network Organization (http://www.netforex.org/)
  314. '/forex trading network organization/i',
  315. '/info@netforex\.org/i',
  316. // FunnelBack (http://www.funnelback.com/)
  317. '/funnelback/i',
  318. // Baidu (http://www.baidu.com/)
  319. '/baiduspider/i',
  320. // Brandimensions (http://www.brandimensions.com/)
  321. '/bdfetch/i',
  322. // Blaiz Enterprises (http://www.blaiz.net/)
  323. '/blaiz-bee/i',
  324. // Boitho/SearchDaimon (http://www.boitho.com/ or http://www.searchdaimon.com/)
  325. '/boitho\.com-dc/i',
  326. // Celestial (OAI aggregator, see http://oai-perl.sourceforge.net/ for a little info)
  327. '/celestial/i',
  328. // Cipinet (http://www.cipinet.com/)
  329. '/cipinetbot/i',
  330. // iVia (http://ivia.ucr.edu/)
  331. '/crawlertest crawlertest/i',
  332. // Encyclopedia of Keywords (http://keywen.com/)
  333. '/easydl/i',
  334. // Everest-Vulcan Inc. (http://everest.vulcan.com/)
  335. '/everest-vulcan inc/i',
  336. // FactBites (http://www.factbites.com/)
  337. '/factbot/i',
  338. // Scirus (http://www.scirus.com/)
  339. '/scirus scirus-crawler@fast\.no/i',
  340. // UOL (http://www.uol.com.br/)
  341. '/uolcrawler/i',
  342. '/soscrawler@uol\.com\.br/i',
  343. // Always Updated (http://www.updated.com/)
  344. '/updated crawler/i',
  345. '/crawler@updated\.com/i',
  346. // FAST Enterprise Search (http://www.fast.no/)
  347. '/fast metaweb crawler/i',
  348. '/crawler@fast\.no/i',
  349. '/helpdesk at fastsearch dot com/i',
  350. // Deutsche Wortschatz Portal (http://wortschatz.uni-leipzig.de/)
  351. '/findlinks/i',
  352. // Gais (http://gais.cs.ccu.edu.tw/)
  353. '/gaisbot/i',
  354. '/robot[0-9]{2}@gais.cs.ccu.edu.tw/i',
  355. // http://ilse.net/
  356. '/ingrid/i',
  357. // Krugle (http://corp.krugle.com/)
  358. '/krugle\/krugle/i',
  359. '/krugle web crawler/i',
  360. '/webcrawler@krugle\.com/i',
  361. // WebWobot (http://www.webwobot.com/)
  362. '/scollspider/i',
  363. // Omni-Explorer (http://www.omni-explorer.com/)
  364. '/omniexplorer_bot/i',
  365. '/worldindexer/i',
  366. // PageBull (http://www.pagebull.com/)
  367. '/pagebull http:\/\/www\.pagebull\.com\//i',
  368. // dir.com (http://dir.com/)
  369. '/pompos/i',
  370. // Sensis (http://sensis.com.au/)
  371. '/sensis web crawler/i',
  372. '/search_comments\\\\at\\\\sensis\\\\dot\\\\com\\\\dot\\\\au/i',
  373. // Shopwiki (http://www.shopwiki.com/)
  374. '/shopwiki/i',
  375. // Guruji (http://www.terrawiz.com/)
  376. '/terrawizbot/i',
  377. // Language Observatory Project (http://www.language-observatory.org/)
  378. '/ubicrawler/i',
  379. // MSIE offline bookmarks crawler
  380. '/msiecrawler/i',
  381. // Unidentified
  382. '/bot/i',
  383. '/crawler/i',
  384. '/spider/i',
  385. '/larbin/i', // also larbinSpider
  386. '/httrack/i',
  387. '/voyager/i',
  388. '/acadiauniversitywebcensusclient/i',
  389. '/feedchecker/i',
  390. '/knowitall\(knowitall@cs\.washington\.edu\)/i',
  391. '/mediapartners-google/i',
  392. '/psycheclone/i',
  393. '/topicblogs/i',
  394. '/nutch/i',
  395. ),
  396. );
  397.  
  398. ###########################################
  399. ##
  400. ## No configuration required below here.
  401. ##
  402. ###########################################
  403.  
  404. $connect = mysql_pconnect ($sqlserver,$sqluser,$sqlpass);
  405. $db = mysql_select_db($sqldatabase,$connect) or die("Could not connect");
  406.  
  407. // First get the date of last update
  408. /* NJS 2006-04-28
  409. Changed this from order by timeinsert to order by id. The ID is
  410. always guaranteed to increase temporally, but is otherwise
  411. time-independent and thus not affected by things like daylight
  412. savings.
  413. */
  414. $query = "SELECT lastproc FROM lastproc ORDER BY id DESC LIMIT 1";
  415. $result = mysql_query($query,$connect);
  416. $num_rows = mysql_num_rows($result);
  417. if ($num_rows > 0) {
  418. $row = mysql_fetch_assoc($result);
  419. $lastproc = $row["lastproc"];
  420. // NJS 2007-01-30 Refactored $datetestA to more meaningful $test_lastproc.
  421. $test_lastproc = strtotime($lastproc);
  422. }
  423. else {
  424. $test_lastproc = 0;
  425. }
  426.  
  427. // NJS 2006-06-14: Generalised connection list for multiple archives.
  428. $eprints_connections = array();
  429. foreach ($eprintsdbs as $archive_name => $details)
  430. {
  431. $eprints_connections[$archive_name] =
  432. mysql_connect($details['sqlserver'],$details['username'],$details['password']);
  433. }
  434. $counter = 0;
  435. foreach($log_file as $archive_name=>$archive_log) {
  436. $logf = $log_dir . $archive_log;
  437. $handle = fopen($logf, "r");
  438. while (!feof($handle)) {
  439. $buffer = fgets($handle, 4096);
  440. /* NJS 2007-01-26
  441. Added user-agent match to all regexps to enable bot detection.
  442. NJS 2007-01-31
  443. Refactored regexps from four down to one, after realising
  444. that (a) long EPrints URLs are a superset of the short ones,
  445. and (b) a regexp that matches domain names works just as well
  446. for IP addresses (the GeoIP lookup doesn't care which it
  447. gets). Also fixed the pattern so it can handle an arbitrary
  448. number of subdomains. Note that the latter would be the main
  449. argument for keeping a separate IP address pattern, as IP
  450. addresses always comprise exactly four parts. However, it's
  451. not really up to the script to verify IP addresses; Apache
  452. should be recording them correctly in the first place!
  453. The typical kinds of strings we are matching look something
  454. like this:
  455. fetch abstract (short, long):
  456. 168.192.1.1 - - [31/Jan/2007:09:15:36 +1300] "GET /1/ HTTP/1.1" 200 12345 "referer" "user-agent"
  457. 168.192.1.1 - - [31/Jan/2007:09:15:36 +1300] "GET /archive/00000001/ HTTP/1.1" 200 12345 "referer" "user-agent"
  458. download item (short, long):
  459. 168.192.1.1 - - [31/Jan/2007:09:15:37 +1300] "GET /1/01/foo.pdf HTTP/1.1" 200 12345 "referer" "user-agent"
  460. 168.192.1.1 - - [31/Jan/2007:09:15:37 +1300] "GET /archive/00000001/01/foo.pdf HTTP/1.1" 200 12345 "referer" "user-agent"
  461. Plus any of the above with a domain name substituted for the IP
  462. address (e.g., foo.bar.com instead of 168.192.1.1).
  463. */
  464. if (preg_match("/^(\S+(?:\.\S+)+) - - \[(.*?)\] \"GET \/(?:archive\/0+)?(\d+).*? HTTP\/1..\" 200 .*?(\"[^\"]+\")?$/i",$buffer,$matches))
  465. {
  466. $counter++;
  467. $country_code = '';
  468. $country_name = '';
  469. $insertid = '';
  470. $eprint_name = '';
  471. $view_type = '';
  472. $uniquebits = '';
  473. /* NJS 2007-01-29
  474. Moved date checking to the start of the loop, as there's
  475. no point in doing any of the regexp checks if we've already
  476. processed this log entry and will discard it anyway.
  477. */
  478. $date = $matches[2];
  479. /* NJS 2006-04-28
  480. Switched to timestamp rather than date-based comparison.
  481. First, clean up the Apache request date into something
  482. that strtotime understands. Note that the Apache log
  483. dates include time zone info by default.
  484. */
  485. $date = preg_replace("/:/"," ",$date,1); // Change first ":" to " ".
  486. $date = preg_replace("/\//", " ", $date); // Change all "/" to " ".
  487. // NJS 2007-01-30 Refactored $databaseB to more meaningful
  488. // $test_logdate.
  489. $test_logdate = strtotime($date);
  490.  
  491. // NJS 2007-01-30 Added test for log dates >= $start_time.
  492. if ( ( $test_logdate < $test_lastproc ) ||
  493. ( $test_logdate >= $test_starttime ) )
  494. continue;
  495. // Convert to properly formatted date string.
  496. $request_date = date('Y-m-d H:i:s O', $test_logdate);
  497. /* NJS 2005-12-16
  498. Determine country code and name.
  499. Check whether the IP number falls into any of the local
  500. intranet ranges. If so, then use that.
  501. */
  502. $ip = $matches[1];
  503. $ip_long = ip2long($ip);
  504. $found_country = FALSE;
  505. foreach ($local_IPs as $id => $addresses)
  506. {
  507. foreach ($addresses as $ip_range)
  508. {
  509. if (is_array($ip_range)) // check against lower/upper bounds
  510. {
  511. $found_country = (($ip_long >= $ip_range['lower'])
  512. && ($ip_long <= $ip_range['upper']));
  513. }
  514. else if (is_long($ip_range)) // data type sanity check
  515. {
  516. $found_country = ($ip_long == $ip_range);
  517. }
  518. else // something is seriously broken, ignore this entry
  519. {
  520. print "Unsupported data type " . gettype($ip_range) .
  521. " (value " . $ip_range .
  522. ") in \$local_IPs (expected long).\n";
  523. continue;
  524. }
  525. if ( $found_country ) break;
  526. }
  527. if ($found_country)
  528. {
  529. list($country_name, $country_code) = explode(',', $id);
  530. break;
  531. }
  532. }
  533. // Otherwise, fall back to GeoIP.
  534. if (!$found_country)
  535. {
  536. $country_code = geoip_country_code_by_addr($gi, $ip);
  537. $country_name = geoip_country_name_by_addr($gi, $ip);
  538. }
  539. // end NJS 2005-12-16
  540. /* NJS 2007-01-26
  541. Check whether this is a bot reference.
  542. */
  543. $user_agent = count($matches) > 4 ? $matches[4] : '';
  544. $found_country = FALSE;
  545. foreach ($bot_patterns as $id => $patterns)
  546. {
  547. foreach ($patterns as $pat)
  548. {
  549. if (preg_match($pat, $user_agent))
  550. {
  551. $found_country = TRUE;
  552. break;
  553. }
  554. }
  555. if ($found_country)
  556. {
  557. list($country_name, $country_code) = explode(',', $id);
  558. break;
  559. }
  560. }
  561. // end NJS 2007-01-26
  562. // Now sort out the remaining bits and we're done.
  563. $eprint_id = $matches[3];
  564. $uniquebits = $buffer;
  565. // NJS 2005-11-25 Added regexp for EPrints short URLs.
  566. // NJS 2007-01-31 Refactored into one regexp for both styles.
  567. if (preg_match("/GET \/(?:archive\/0+)?\d+\/\d+\//i",$buffer)) {
  568. $view_type = "download";
  569. } else {
  570. $view_type = "abstract";
  571. }
  572. if(isset($eprintname[$archive_name . $eprint_id])) {
  573. $eprint_name = $eprintname[$archive_name . $eprint_id];
  574. } else {
  575. $eprint_name = getePrintName($eprints_connections[$archive_name],$eprintsdbs[$archive_name]['dbname'],$eprint_id,$eprints_version);
  576. $eprintname[$archive_name . $eprint_id] = $eprint_name;
  577. }
  578. if($eprint_name=='') {
  579. // Do nothing.
  580. } else {
  581. $eprint_name = mysql_escape_string($eprint_name);
  582. /* NJS 2006-04-25
  583. Requests containing apostrophes (') are dumped by
  584. MySQL unless we escape them. Looking in the GeoIP
  585. files I also see country names with apostrophes, so
  586. escape that as well. Everything else should be fine.
  587. */
  588. $uniquebits = mysql_escape_string($uniquebits);
  589. $country_name = mysql_escape_string($country_name);
  590. // end NJS 2006-04-25
  591. $query = "
  592. INSERT INTO view (uniquebits,archive_name,ip,request_date,archiveid,country_code,country_name,view_type,eprint_name)
  593. VALUES('".$uniquebits."','".$archive_name."','".$ip."','".$request_date."',".$eprint_id.",'".$country_code."','".$country_name."','".$view_type."','".$eprint_name."')";
  594. $result = mysql_query($query,$connect);
  595. $insertid = mysql_insert_id($connect);
  596. }
  597.  
  598. } else {
  599. // print "NO match" . "\n";
  600. }
  601. }
  602. fclose($handle);
  603. }
  604.  
  605. /*
  606. Keep track of where we are. Should avoid duplication of results
  607. if the script is run more than once on the same log file.
  608. */
  609.  
  610. // NJS 2006-04-28 Switched value inserted to $start_time instead of $request_date.
  611. $query = "INSERT into lastproc (lastproc) values('".$start_time."')";
  612. $result = mysql_query($query,$connect);
  613.  
  614. #print "Records counted: $counter\n";
  615. #print "Last count: $request_date\n";
  616. foreach ($eprints_connections as $connection)
  617. {
  618. mysql_close($connection);
  619. }
  620. mysql_close($connect);
  621.  
  622. // Look up the title corresponding to the specified eprint id.
  623. function getePrintName($connection,$dbname,$eprintid,$eprints_version) {
  624. // NJS 2006-06-14: DB connection now passed as an argument.
  625. $db = mysql_select_db($dbname,$connection);
  626. // NJS 2007-07-24: Added check for EPrints version, as the
  627. // database structure changed between versions 2 and 3.
  628. if ( $eprints_version > 2 )
  629. {
  630. $query3 = "
  631. SELECT title
  632. FROM eprint
  633. WHERE eprintid = $eprintid
  634. AND eprint_status = 'archive'
  635. ";
  636. }
  637. else
  638. {
  639. $query3 = "
  640. SELECT title
  641. FROM archive
  642. WHERE eprintid = $eprintid
  643. ";
  644. }
  645. $result3 = mysql_query($query3,$connection);
  646. $title = '';
  647. $suffix = '';
  648. // NJS 2006-04-25 Added check for empty result, probably a deleted item.
  649. // Look in the deleted items for details.
  650. if (mysql_num_rows($result3) == 0) {
  651. // NJS 2007-07-24: Added check for EPrints version, as the
  652. // database structure changed between versions 2 and 3.
  653. if ( $eprints_version > 2 )
  654. {
  655. $query3 = "
  656. SELECT title
  657. FROM eprint
  658. WHERE eprintid = $eprintid
  659. AND eprint_status = 'deletion'
  660. ";
  661. }
  662. else
  663. {
  664. $query3 = "
  665. SELECT title
  666. FROM deletion
  667. WHERE eprintid = $eprintid
  668. ";
  669. }
  670. $result3 = mysql_query($query3,$connection);
  671. // If it's not in deletion, then we have no clue what it is.
  672. if (mysql_num_rows($result3) == 0) {
  673. $title = "Unknown item [$eprintid]";
  674. }
  675. else {
  676. $suffix = ' [deleted]';
  677. }
  678. }
  679. if ($title == '') {
  680. $row = mysql_fetch_assoc($result3);
  681. $row["title"] = trim($row["title"]);
  682. $row["title"] = preg_replace("/\s+/"," ",$row["title"]);
  683. $title = $row["title"];
  684. }
  685. return $title . $suffix;
  686. }
  687.  
  688. ?>
  689.