Тянем дневные данные с Yahoo.Finance: код на Java

package com.cognitum.research.data;

import java.io.*;

import java.net.URL;

import java.net.URLEncoder;

import java.text.SimpleDateFormat;

import java.util.*;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

public class YahooData {

public static class YBar {

public final Date date;

public final double open;

public final double high;

public final double low;

public final double close;

public final double volume;

public final double adjusted;

public YBar(Date date, double open, double high, double low, double close, double volume, double adjusted) {

this.date = date;

this.open = open;

this.high = high;

this.low = low;

this.close = close;

this.volume = volume;

this.adjusted = adjusted;

}

@Override

public String toString() {

return String.format(Locale.US, “%s,%f,%f,%f,%f,%f,%f”,

dateFormat.format(date), open, high, low, close, volume, adjusted);

}

}

public final static SimpleDateFormat dateFormat = new SimpleDateFormat(“yyyyMMdd”);

public final Map<Date, YBar> bars = new TreeMap<>();

public YahooData(String ticker) {

String encodedTicker = ticker;

try {

encodedTicker = URLEncoder.encode(ticker, “UTF-8”);

} catch (UnsupportedEncodingException e) {

e.printStackTrace();

}

// GETTING DATA FROM “DOWNLOAD HISTORICAL DATA” SOURCE

System.out.println(“getting data from the “download to spreadsheet” source…”);

final Calendar tm = Calendar.getInstance();

tm.add(Calendar.DATE, 1);

final String url = String.format(

“http://ichart.finance.yahoo.com/table.csv?s=%s&d=%d&e=%d&f=%d&g=d&a=&b=&c=&ignore=.csv&tmstmp=%d”,

encodedTicker, tm.get(Calendar.MONTH), tm.get(Calendar.DATE), tm.get(Calendar.YEAR), tm.getTimeInMillis());

final SimpleDateFormat df = new SimpleDateFormat(“yyyy-MM-dd”);

try {

for (String s : getUrlStrings(url)) {

try {

final String[] ss = s.trim().split(“,”);

final Date d = df.parse(ss[0]);

final double open = Double.valueOf(ss[1]);

final double high = Double.valueOf(ss[2]);

final double low = Double.valueOf(ss[3]);

final double close = Double.valueOf(ss[4]);

final double volume = Double.valueOf(ss[5]);

final double adjusted = Double.valueOf(ss[6]);

bars.put(d, new YBar(d, open, high, low, close, volume, adjusted));

} catch (Exception e) {

System.out.println(“cannot parse ” + s);

}

}

} catch (Exception e) {

System.out.println(“cannot get data from ” + url);

}

// GETTING DATA FROM THE WEB TABLE

System.out.println(“getting data from the web table…”);

final String url2 = String.format(“http://finance.yahoo.com/q/hp?s=%s+Historical+Prices”, encodedTicker);

final SimpleDateFormat df2 = new SimpleDateFormat(“MMM d, yyyy”);

try {

final StringBuilder sb = new StringBuilder();

for (String s : getUrlStrings(url2)) {

sb.append(s.trim());

}

final Pattern p = Pattern.compile(“<tr>” +

“<td class.*?>(.*?)</td>” +

“<td .*?>(.*?)</td>” +

“<td .*?>(.*?)</td>” +

“<td .*?>(.*?)</td>” +

“<td .*?>(.*?)</td>” +

“<td .*?>(.*?)</td>” +

“<td .*?>(.*?)</td>” +

“</tr>”);

final Matcher m = p.matcher(sb.toString());

while (m.find()) {

try {

final Date dt = df2.parse(m.group(1));

final double open = Double.valueOf(m.group(2).replace(“,”, “”));

final double high = Double.valueOf(m.group(3).replace(“,”, “”));

final double low = Double.valueOf(m.group(4).replace(“,”, “”));

final double close = Double.valueOf(m.group(5).replace(“,”, “”));

final double volume = Double.valueOf(m.group(6).replace(“,”, “”));

final double adjusted = Double.valueOf(m.group(7).replace(“,”, “”));

bars.put(dt, new YBar(dt, open, high, low, close, volume, adjusted));

} catch (Exception e) {

System.out.println(“cannot parse string: ” + m.group());

}

}

} catch (Exception e) {

System.out.println(“cannot get data from ” + url2);

}

// GETTING LAST DATA FROM THE MAIN TICKER PAGE

System.out.println(“getting last data from the main ticker page…”);

final String url3 = String.format(“http://download.finance.yahoo.com/d/quotes.csv?s=%s&f=sl1d1t1c1ohgv&e=.csv”, encodedTicker);

final SimpleDateFormat df3 = new SimpleDateFormat(“MM/dd/yyyy”);

try {

final String currentString = getUrlStrings(url3).get(0).trim();

try {

final String[] ss = currentString.split(“,”);

final Date d = df3.parse(ss[2].replace(“””, “”));

final double open = Double.valueOf(ss[5]);

final double high = Double.valueOf(ss[6]);

final double low = Double.valueOf(ss[7]);

final double close = Double.valueOf(ss[1]);

final double volume = Double.valueOf(ss[8]);

if (!bars.containsKey(d)) bars.put(d, new YBar(d, open, high, low, close, volume, close));

} catch (Exception e) {

System.out.println(“cannot parse ” + currentString);

}

} catch (Exception e) {

System.out.println(“cannot get data from ” + url3);

}

}

public void toFile(String fileName) {

try (final BufferedWriter bw = new BufferedWriter(new FileWriter(fileName))) {

for (YBar b : bars.values()) bw.write(b.toString() + “n”);

} catch (IOException e) {

e.printStackTrace();

}

}

public static List<String> getUrlStrings(String urlString) {

final List<String> list = new ArrayList<>();

try (final BufferedReader br = new BufferedReader(new InputStreamReader(

new URL(urlString).openConnection().getInputStream()))) {

String inputLine;

while ((inputLine = br.readLine()) != null) list.add(inputLine);

} catch (Exception e) {

e.printStackTrace();

}

return list;

}

public static void main(String[] args) {

final YahooData data = new YahooData(“SPY”);

System.out.println(“downloaded bars: ” + data.bars.size());

data.toFile(“c:/test/bars/yahoo-data-test.txt”);

}

}