twitter_batch.js

// Generated by IcedCoffeeScript 108.0.11
(function() {
  var BaseBearerToken, BaseScraper, Lock, TweetCache, TwitterBatchScraper, TwitterScraper, constants, decode_sig, iced, make_ids, schema, sncmp, urlmod, v_codes, ws_normalize, __iced_k, __iced_k_noop, _ref,
    __hasProp = {}.hasOwnProperty,
    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };

  iced = require('iced-runtime');
  __iced_k = __iced_k_noop = function() {};

  _ref = require('./base'), sncmp = _ref.sncmp, BaseScraper = _ref.BaseScraper, BaseBearerToken = _ref.BaseBearerToken;

  TwitterScraper = require('./twitter').TwitterScraper;

  make_ids = require('../base').make_ids;

  constants = require('../constants').constants;

  v_codes = constants.v_codes;

  decode_sig = require('kbpgp').ukm.decode_sig;

  Lock = require('../util').Lock;

  urlmod = require('url');

  schema = require('../schema3');

  ws_normalize = function(x) {
    var v;
    v = x.split(/[\t\r\n ]+/);
    if (v.length && v[0].length === 0) {
      v.shift();
    }
    if (v.length && v.slice(-1)[0].length === 0) {
      v.pop();
    }
    return v.join(' ');
  };

  exports.TwitterBatchScraper = TwitterBatchScraper = (function(_super) {
    __extends(TwitterBatchScraper, _super);

    function TwitterBatchScraper(opts) {
      this._tweet_cache = opts.tweet_cache;
      this.cache_refresh_interval = opts.cache_refresh_interval;
      TwitterBatchScraper.__super__.constructor.call(this, opts);
    }

    TwitterBatchScraper.prototype._hunt_batch = function(cb) {
      var created_at, err, i, id, json, query, rc, since_id, text, u, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
      __iced_k = __iced_k_noop;
      ___iced_passed_deferral = iced.findDeferral(arguments);
      query = {
        query: "\"Verifying myself\" \"Keybase.io\"",
        expansions: "author_screen_name",
        "user.fields": "url,username",
        "tweet.fields": "created_at",
        max_results: 60
      };
      if (since_id = this._tweet_cache.last_id) {
        query.since_id = since_id;
      }
      u = urlmod.format({
        host: "api.twitter.com",
        protocol: "https:",
        pathname: "/2/tweets/search/recent",
        query: query
      });
      (function(_this) {
        return (function(__iced_k) {
          __iced_deferrals = new iced.Deferrals(__iced_k, {
            parent: ___iced_passed_deferral,
            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter_batch.iced",
            funcname: "TwitterBatchScraper._hunt_batch"
          });
          _this._get_body_api({
            url: u
          }, __iced_deferrals.defer({
            assign_fn: (function() {
              return function() {
                err = arguments[0];
                rc = arguments[1];
                return json = arguments[2];
              };
            })(),
            lineno: 44
          }));
          __iced_deferrals._fulfill();
        });
      })(this)((function(_this) {
        return function() {
          var _i, _len, _ref1, _ref2;
          _this.log("| search index " + u + " -> " + rc);
          if (rc !== v_codes.OK) {

          } else if ((typeof json === "undefined" || json === null) || (json.length === 0)) {
            rc = v_codes.EMPTY_JSON;
          } else if (json.data == null) {
            rc = v_codes.INVALID_JSON;
          } else {
            console.log(json.data);
            _ref1 = json.data;
            for (i = _i = 0, _len = _ref1.length; _i < _len; i = ++_i) {
              _ref2 = _ref1[i], id = _ref2.id, created_at = _ref2.created_at, username = _ref2.username, text = _ref2.text;
              created_at = new Date(created_at);
              if (!isFinite(created_at)) {
                _this.log("got invalid date in tweet JSON id: " + id + ", created_at: " + tweet.created_at);
                continue;
              }
              _this.log("ingesting tweet: id: " + id + ", username: " + username + ", text: \"" + text + "\"");
              _this._tweet_cache.inform({
                id: id,
                created_at: created_at,
                username: username,
                text: text
              });
            }
          }
          return cb(null, v_codes.OK);
        };
      })(this));
    };

    TwitterBatchScraper.prototype.hunt2 = function(_arg, cb) {
      var api_url, current_tweet, err, human_url, name, now, out, proof_text_check, rc, remote_id, username, ___iced_passed_deferral, __iced_deferrals, __iced_k;
      __iced_k = __iced_k_noop;
      ___iced_passed_deferral = iced.findDeferral(arguments);
      username = _arg.username, name = _arg.name, proof_text_check = _arg.proof_text_check;
      (function(_this) {
        return (function(__iced_k) {
          __iced_deferrals = new iced.Deferrals(__iced_k, {
            parent: ___iced_passed_deferral,
            filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter_batch.iced",
            funcname: "TwitterBatchScraper.hunt2"
          });
          _this._tweet_cache.lock.acquire(__iced_deferrals.defer({
            lineno: 63
          }));
          __iced_deferrals._fulfill();
        });
      })(this)((function(_this) {
        return function() {
          err = null;
          now = Math.floor(Date.now() / 1000);
          (function(__iced_k) {
            if (now - _this._tweet_cache.fetched_at > _this.cache_refresh_interval) {
              _this._tweet_cache.fetched_at = now;
              (function(__iced_k) {
                __iced_deferrals = new iced.Deferrals(__iced_k, {
                  parent: ___iced_passed_deferral,
                  filename: "/Users/michal/SourceCode/keybase/go/src/github.com/keybase/server_test_progs/proofs/src/scrapers/twitter_batch.iced",
                  funcname: "TwitterBatchScraper.hunt2"
                });
                _this._hunt_batch(__iced_deferrals.defer({
                  assign_fn: (function() {
                    return function() {
                      err = arguments[0];
                      return rc = arguments[1];
                    };
                  })(),
                  lineno: 68
                }));
                __iced_deferrals._fulfill();
              })(function() {
                return __iced_k(!err && rc !== v_codes.OK ? err = new Error("rc: " + rc) : void 0);
              });
            } else {
              return __iced_k();
            }
          })(function() {
            _this._tweet_cache.lock.release();
            if (err) {
              _this.logl("error", "error when hunting batch: " + (err.toString()));
              return cb(err);
            }
            out = {};
            rc = v_codes.NOT_FOUND;
            current_tweet = _this._tweet_cache.tweets.get(username);
            if (current_tweet && (_this.find_sig_in_tweet({
              inside: current_tweet.text,
              proof_text_check: proof_text_check
            })) === v_codes.OK) {
              rc = v_codes.OK;
              remote_id = current_tweet.id;
              api_url = human_url = _this._id_to_url(username, remote_id);
              out = {
                remote_id: remote_id,
                api_url: api_url,
                human_url: human_url
              };
            }
            out.rc = rc;
            return cb(err, out);
          });
        };
      })(this));
    };

    return TwitterBatchScraper;

  })(TwitterScraper);

  exports.TweetCache = TweetCache = (function() {
    function TweetCache() {
      this.tweets = new Map();
      this.last_id = null;
      this.fetched_at = 0;
      this.lock = new Lock();
    }

    TweetCache.prototype.inform = function(_arg) {
      var created_at, current, id, text, username;
      id = _arg.id, created_at = _arg.created_at, username = _arg.username, text = _arg.text;
      current = this.tweets.get(username);
      if (current && current.created_at >= created_at) {
        return;
      }
      return this.tweets.set(username, {
        id: id,
        created_at: created_at,
        text: text
      });
    };

    return TweetCache;

  })();

}).call(this);