Scrape only new content

// Create a file object that will point to the file containing
// the search terms.
File inputFile = new File( "jogadores.txt" );

// These two objects are needed to read the file.
FileReader in = new FileReader( inputFile );
BufferedReader buffRead = new BufferedReader( in );

// Read the file in line-by-line.  Each line in the text file
// will contain a search term.
while( ( url_jogador = buffRead.readLine() )!=null)
{
    // Set a session variable corresponding to the search term.
    session.setVariable( "url_jogador", url_jogador );

    // Get search results for this particular search term.
    session.scrapeFile( "Stats" );
}

// Close up the objects to indicate we're done reading the file.
in.close();
buffRead.close();



The extractor pattern matches a single row:
<tr class="~@odd@~">
         <td class="dt">~@data@~</td>
         <td class="op"><span class="atvs">~@local@~</span><a href="/~@oponente@~/" target="_parent">~@sigla_oponente@~</a></td>
         <td class="rs"><a target="_parent" href="~@box_score@~">~@resultado@~</a></td>
         <td>~@minutos@~</td>
         <td>~@fg@~</td>
         <td>~@3p@~</td>
         <td>~@ft@~</td>
         <td>~@off@~</td>
         <td>~@def@~</td>
         <td>~@rebotes@~</td>
         <td>~@assists@~</td>
         <td>~@steals@~</td>
         <td>~@blocks@~</td>
         <td>~@turnovers@~</td>
         <td>~@pf@~</td>
         <td>~@pontos@~</td>
      </tr>



Then, after each pattern match, I run this script to save in a mysql db:
Class.forName("com.mysql.jdbc.Driver").newInstance();
      Connection conn;
      conn = DriverManager.getConnection("jdbc:mysql://" + session.getVariable("MYSQL_SERVER_URL") + ":"+session.getVariable("MYSQL_SERVER_PORT") + "/" +  session.getVariable("MYSQL_DATABASE"), session.getVariable("MYSQL_SERVER_USER"), session.getVariable("MYSQL_SERVER_PASSWORD"));

nome = session.getVariable( "nome" );
data = dataRecord.get( "data" );
local = dataRecord.get( "local" );
oponente = dataRecord.get( "oponente" );
result = dataRecord.get( "resultado" );
result = result.substring(0,1);
minutos = dataRecord.get( "minutos" );
rebotes = dataRecord.get( "rebotes" );
assists = dataRecord.get( "assists" );
steals = dataRecord.get( "steals" );
blocks = dataRecord.get( "blocks" );
turnovers = dataRecord.get( "turnovers" );
pontos = dataRecord.get( "pontos" );

reb = Double.valueOf(rebotes);
reb = reb.doubleValue();
ast = Double.valueOf(assists);
ast = ast.doubleValue();
stl = Double.valueOf(steals);
stl = stl.doubleValue();
blk = Double.valueOf(blocks);
blk = blk.doubleValue();
to = Double.valueOf(turnovers);
to = to.doubleValue();
pts = Double.valueOf(pontos);
pts = pts.doubleValue();
double fp = ((reb+ast)*0.8)+((stl+blk)*1.5)-(to*0.5)+(pts*0.6);
fp = Math.round(fp);
int fps = (int)fp;
fpoints = String.valueOf(fps);

SimpleDateFormat sdf = new SimpleDateFormat("MMM d");
Calendar c = Calendar.getInstance();
c.setTime(sdf.parse(data));
java.sql.Date data2 = new java.sql.Date(sdf.parse(data).getTime());
java.sql.Date outubro = java.sql.Date.valueOf("1970-10-01");
if (data2.after(outubro)){
        c.add(Calendar.YEAR, 41);
}
else {
        c.add(Calendar.YEAR, 42);
}

data = new java.sql.Date(c.getTime().getTime());
java.sql.Date lastScrapedDate = java.sql.Date.valueOf("2012-01-17");

String nome_certo = nome.replaceAll("'","\\\\\'");

if (data.after(lastScrapedDate))
        {
        //Create statements and run queries
        // on your database.
        Statement stmt = null;
        stmt = conn.createStatement();

      mysqlstring="INSERT IGNORE INTO jogos VALUES('"+nome_certo+"','"+data+"','"+local+"','"+oponente+"','"+result+"','"+minutos+"','"+rebotes+"','"+assists+"','"+steals+"','"+blocks+"','"+turnovers+"','"+pontos+"','"+fpoints+"')";
      stmt.executeUpdate(mysqlstring);

        //Be sure to close up your
        // statements and connection.
        stmt.close();
 }
 else {
        //skip
 }
conn.close();