Extract Google Blog Export using Java

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

public class BlogSaver
{
  public static void main(String[] args) throws Exception {
    System.err.println("Working in "+System.getProperty("user.dir"));
    if (args.length != 1)
      syntax();
    
    final String fileName = args[0];
    final String basePath = new File(fileName).getParent();
    final BlogXmlHandler handler = new BlogXmlHandler(basePath == null ? "." : basePath);
    final SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
    parser.parse(new BufferedInputStream(new FileInputStream(fileName)), handler);
  }
  
  private static void syntax() {
    System.err.println("SYNTAX: java "+BlogSaver.class.getName()+" blog-exportfile.xml");
    System.exit(1);
  }

  private static class BlogXmlHandler extends DefaultHandler
  {
    private final String basePath;
    private final StringBuffer characters = new StringBuffer();
    
    private boolean collectText;
    private boolean wantThis;
    private String published;
    private String updated;
    private String title;

    public BlogXmlHandler(String basePath) {
      this.basePath = basePath;
    }

    @Override
    public void characters(char [] chars, int start, int length) throws SAXException {
      if (collectText)
        characters.append(chars, start, length);
    }

    @Override
    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
      if (qName.equals("entry")) {
        collectText = true;
        wantThis = false;
      }
      else if (qName.equals("category")) {
        wantThis = attributes.getValue("term").endsWith("kind#post");
      }
      characters.setLength(0);
    }

    @Override
    public void endElement(String uri, String localName, String qName) throws SAXException {
      if (collectText)
        if (qName.equals("entry"))
          collectText = false;
        else if (qName.equals("published"))
          published = characters.toString();
        else if (qName.equals("updated"))
          updated = characters.toString();
        else if (qName.equals("title"))
          title = characters.toString().trim();
        else if (wantThis && qName.equals("content"))
          process(characters.toString());
    }
    
    private void process(String blogPost) {
      HtmlWriter page = null;
      try {
        page = new HtmlWriter(toFileName(title), title, published, updated);
        page.writeHtml(blogPost);
      }
      catch (IOException ex) {
        throw new RuntimeException(ex);
      }
      finally {
        try {
          if (page != null)
            page.close();
        }
        catch (Exception ex) { // ignore close error
        }
      }
    }
    
    private String toFileName(String title) {
      final String path = basePath + "/" + "blog";
      new File(path).mkdirs();
      final int maxLength = 64 - ".html".length();
      title = replace(title);
      title = (title.length() <= maxLength) ? title : title.substring(0, maxLength);
      final String fileName = path+"/"+title;
      return fileName+".html";
    }
    
    private String replace(String title) {
      final StringBuffer sb = new StringBuffer();
      for (int i = 0; i < title.length(); i++) {
        final char c = title.charAt(i);
        sb.append(Character.isLetterOrDigit(c) ? c : '_');
      }
      return sb.toString();
    }
  }
  
  private static class HtmlWriter extends FileWriter
  {
    private final String pagePath;
    private final String pageName;
    private final boolean noSmallImagePreview = true;
    
    public HtmlWriter(String fileName, String title, String published, String updated) throws IOException {
      super(fileName);
      
      final File file = new File(fileName);
      pagePath = file.getParent();
      pageName = file.getName();
      
      write("<!DOCTYPE html>\n");
      write("<html>\n");
      write("<head>\n");
      write("  <meta charset=\"UTF-8\" />\n");
      write("  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"/ />\n");
      write("  <title>"+title+"</title>\n");
      write("</head>\n");
      write("<body>\n");
      write("  <h1 style='text-align: center;'>"+title+"</h1>\n");
      write("  <hr/>");
      write("  <p>Published: "+published.substring(0, 10)+"<br/>\n");
      write("     Updated: "+updated.substring(0, 10)+"</p>\n");
      write("  <hr/>");
    }
    
    public void writeHtml(String html) throws IOException {
      // handle all google image URLs by downloading the image
      // to a local folder and replacing the URL by a local URL
      final Pattern pattern = Pattern.compile(
        noSmallImagePreview ?
          "<a href=\"(https?://[^>]*\\.bp\\.blogspot\\.com/[^\"]+)\"[^>]*><img[^>]*></a>"
        :
          "=\"(https?://[^>]*\\.bp\\.blogspot\\.com/[^\"]+)\""
      );
      final Matcher matcher = pattern.matcher(html);
      final StringBuffer sb = new StringBuffer();
      while (matcher.find()) {
        final String imageUrl = matcher.group(1);
        matcher.appendReplacement(
          sb,
          noSmallImagePreview ?
            "<img src=\""+saveImage(imageUrl)+"\"/>"
          :
            "=\""+saveImage(imageUrl)+"\""
        );
      }
      matcher.appendTail(sb);
      html = sb.toString();
      
      // write HTML to file
      write(html);
    }
    
    @Override
    public void close() throws IOException {
      write("</body>\n");
      write("</html>\n");
      super.close(); //To change body of generated methods, choose Tools | Templates.
    }
    
    private String saveImage(String urlString) {
      final String imageDir = pageName.substring(0, pageName.length() - ".html".length());
      
      final int lastSlash = urlString.lastIndexOf('/');
      final String firstUrlPart = urlString.substring(0, lastSlash);
      final String lastSubdir = firstUrlPart.substring(firstUrlPart.lastIndexOf('/') + 1);
      final String fileName = urlString.substring(lastSlash + 1);
      
      final File imagePath = new File(new File(pagePath, imageDir), lastSubdir);
      imagePath.mkdirs();
      
      try {
        final URL url = new URL(urlString);
        final File imageFile = new File(imagePath, fileName);
        final FileOutputStream output = new FileOutputStream(imageFile);
        final ReadableByteChannel byteChannel = Channels.newChannel(url.openStream());
        output.getChannel().transferFrom(byteChannel, 0, Long.MAX_VALUE);
        
        final String newUrl = imageDir+"/"+lastSubdir+"/"+fileName;
        System.err.println("Saved image "+newUrl);
        return newUrl;
      }
      catch (MalformedURLException e) {
        throw new RuntimeException(e);
      }
      catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  }

}