Script to import/update the rss aggregation

Gemfile

@@ -10,4 +10,5 @@ gem 'sass'
 gem 'rdiscount'
 gem 't'
 gem 'nokogiri'
+gem 'feedzirra'
 

Gemfile.lock

@@ -2,6 +2,8 @@ GEM
   remote: http://gems.github.com/
   remote: http://rubygems.org/
   specs:
+    activesupport (3.1.12)
+      multi_json (~> 1.0)
     addressable (2.3.3)
     adsf (1.0.1)
       rack (>= 1.0.0)
@@ -11,6 +13,7 @@ GEM
     cookiejar (0.3.0)
     cri (2.3.0)
       colored (>= 1.2)
+    curb (0.7.18)
     daemons (1.1.9)
     em-http-request (1.0.3)
       addressable (>= 2.2.3)
@@ -28,6 +31,16 @@ GEM
     faraday (0.8.5)
       multipart-post (~> 1.1)
     fastercsv (1.5.5)
+    feedzirra (0.1.3)
+      activesupport (~> 3.1.1)
+      builder (>= 2.1.2)
+      curb (~> 0.7.15)
+      i18n (>= 0.5.0)
+      loofah (~> 1.2.0)
+      nokogiri (>= 1.4.4)
+      rake (>= 0.8.7)
+      rdoc (~> 3.8)
+      sax-machine (~> 0.1.0)
     ffi (1.9.0)
     formatador (0.2.4)
     geokit (1.6.5)
@@ -43,12 +56,16 @@ GEM
       nanoc (>= 3.6.3)
     htmlentities (4.3.1)
     http_parser.rb (0.5.3)
+    i18n (0.6.5)
+    json (1.8.0)
     launchy (2.2.0)
       addressable (~> 2.3)
     listen (1.2.3)
       rb-fsevent (>= 0.9.3)
       rb-inotify (>= 0.9)
       rb-kqueue (>= 0.2)
+    loofah (1.2.1)
+      nokogiri (>= 1.4.4)
     lumberjack (1.0.4)
     method_source (0.8.2)
     mini_portile (0.5.1)
@@ -67,14 +84,19 @@ GEM
       method_source (~> 0.8)
       slop (~> 3.4)
     rack (1.4.5)
+    rake (10.1.0)
     rb-fsevent (0.9.3)
     rb-inotify (0.9.0)
       ffi (>= 0.5.0)
     rb-kqueue (0.2.0)
       ffi (>= 0.5.0)
     rdiscount (1.6.8)
+    rdoc (3.12.2)
+      json (~> 1.4)
     retryable (1.3.2)
     sass (3.2.10)
+    sax-machine (0.1.0)
+      nokogiri (> 0.0.0)
     simple_oauth (0.2.0)
     slop (3.4.6)
     systemu (2.5.2)
@@ -108,6 +130,7 @@ PLATFORMS
 DEPENDENCIES
   adsf
   builder
+  feedzirra
   guard-nanoc
   nanoc3
   nokogiri

content/_data/planetcccs-blogroll.yaml

@@ -1,11 +1,11 @@
 blogroll:
   -
-    user: skyr
+    user: Skyr
-    atomfeed: https://stefan.ploing.de/atom.xml
+    feed: https://stefan.ploing.de/atom.xml
   -
     user: princess
-    atomfeed: http://blog.querulantin.de/serendipity/index.php?/feeds/index.atom
+    feed: http://blog.querulantin.de/serendipity/index.php?/feeds/index.atom
   -
     user: Rince
-    atomfeed: http://blog.rince.de/feeds/index.atom
+    feed: http://blog.rince.de/feeds/index.rss2
 

content/planet-cccs.html

@@ -0,0 +1,9 @@
+<% @item[:blogposts].each do |post| %>
+<h2><a href="<%= post[:url] %>"><%= post[:title] %></a></h2>
+<ul class="unstyled inline">
+  <li><i class="icon-calendar"></i> <time itemprop="dateCreated" datetime="<%= post[:date].strftime("%Y-%m-%d") %>"><%= post[:date].strftime("%d.%m.%Y") %></time></li>
+  <li><i class="icon-pencil"></i> <%= post[:user] %></li>
+</ul>
+<p>
+<% end %>
+

content/planet-cccs.md

@@ -1,31 +0,0 @@
------
-title: Planet CCCS
-kind: page
------
-Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy
-eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam
-voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet
-clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit
-amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam
-nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat,
-sed diam voluptua. At vero eos et accusam et justo duo dolores et ea
-rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem
-ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing
-elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna
-aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo
-dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus
-est Lorem ipsum dolor sit amet.
-
-Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse
-molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero
-eros et accumsan et iusto odio dignissim qui blandit praesent luptatum
-zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum
-dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh
-euismod tincidunt ut laoreet dolore magna aliquam erat volutpat.
-
-Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper
-suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem
-vel eum iriure dolor in hendrerit in vulputate velit esse molestie
-consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et
-accumsan et iusto odio dignissim qui blandit praesent luptatum zzril
-delenit augue duis dolore te feugait nulla facilisi.

content/planet-cccs.yaml

@@ -0,0 +1,99 @@
+---
+title: Planet CCCS
+kind: page
+blogposts:
+- user: princess
+  date: 2013-08-23 11:37:25.000000000 Z
+  title: !binary |-
+    TWFjaHQgZXVjaCBmcmVpLi4u
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/355-Macht-euch-frei....html
+- user: Skyr
+  date: 2013-08-18 15:39:52.000000000 Z
+  title: !binary |-
+    VGVsZWZvbmJhbmtpbmctRm5vcmQ=
+  url: http://stefan.ploing.de/2013-08-18-telefonbanking-fnord/
+- user: Skyr
+  date: 2013-08-12 08:49:35.000000000 Z
+  title: !binary |-
+    TWVpbiBQcm9tb3Rpb25zc3plbmFyaW8gLS0gamV0enQgaW4gTG9uZG9u
+  url: http://stefan.ploing.de/2013-08-12-mein-promotionsszenario-in-london/
+- user: princess
+  date: 2013-08-07 15:14:01.000000000 Z
+  title: Vom Tagesausflug zum Höllenritt
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/354-Vom-Tagesausflug-zum-Hoellenritt.html
+- user: princess
+  date: 2013-08-02 10:14:00.000000000 Z
+  title: !binary |-
+    MTMuIEluZm9ybWF0aWNhIEZlbWluYWxlIGFuIGRlciBIUyBGdXJ0d2FuZ2Vu
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/353-13.-Informatica-Feminale-an-der-HS-Furtwangen.html
+- user: Skyr
+  date: 2013-08-02 06:38:34.000000000 Z
+  title: Zwei git-Repositories zusammenführen
+  url: http://stefan.ploing.de/2013-08-02-zwei-git-repos-zusammenfuehren/
+- user: Skyr
+  date: 2013-08-01 11:15:07.000000000 Z
+  title: !binary |-
+    UmFudDogSGFsbG8sIEJsYWNraGF0IT8=
+  url: http://stefan.ploing.de/2013-08-01-rant-hallo-blackhat/
+- user: Skyr
+  date: 2013-07-28 19:09:48.000000000 Z
+  title: Überwachungsstaat - was ist das?
+  url: http://stefan.ploing.de/2013-07-28-ueberwachungsstaat-was-ist-das/
+- user: Skyr
+  date: 2013-07-25 07:48:35.000000000 Z
+  title: !binary |-
+    VG9kby1MaXN0ZSBtaXQgdG9kby50eHQ=
+  url: http://stefan.ploing.de/2013-07-25-todo-liste-mit-todotxt/
+- user: princess
+  date: 2013-07-24 20:35:00.000000000 Z
+  title: !binary |-
+    QXN5bGJld2VyYmVyIHVuZCBBcmJlaXQ=
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/352-Asylbewerber-und-Arbeit.html
+- user: princess
+  date: 2013-07-24 20:26:00.000000000 Z
+  title: Kultusministerium BaWü untersagt dienstlichen Einsatz von sozialen Netzwerken
+    an Schulen
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/351-Kultusministerium-BaWue-untersagt-dienstlichen-Einsatz-von-sozialen-Netzwerken-an-Schulen.html
+- user: princess
+  date: 2013-07-24 20:25:00.000000000 Z
+  title: Bewässerung und Landwirtschaft in Äthiopien
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/350-Bewaesserung-und-Landwirtschaft-in-AEthiopien.html
+- user: Skyr
+  date: 2013-07-20 12:16:14.000000000 Z
+  title: !binary |-
+    RGllIFNwZWljaGVyLU1hdHJqb3NjaGth
+  url: http://stefan.ploing.de/2013-07-20-speicher-matrjoschka/
+- user: princess
+  date: 2013-07-16 21:29:00.000000000 Z
+  title: !binary |-
+    VG91ciBkdXJjaCBkaWUgUGZhbHo=
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/347-Tour-durch-die-Pfalz.html
+- user: princess
+  date: 2013-07-16 20:18:46.000000000 Z
+  title: !binary |-
+    RmFrdCBoZXV0ZSBhYmVuZA==
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/349-Fakt-heute-abend.html
+- user: princess
+  date: 2013-07-15 16:44:45.000000000 Z
+  title: !binary |-
+    TGllZGVyaGFsbGUgLSBiYWNrc3RhZ2UgdW5kIG9uc3RhZ2Ug
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/348-Liederhalle-backstage-und-onstage.html
+- user: Skyr
+  date: 2013-07-12 07:32:35.000000000 Z
+  title: Interessante Keynote über Scala
+  url: http://stefan.ploing.de/2013-07-12-interessante-keynote-ueber-scala/
+- user: princess
+  date: 2013-06-17 16:24:09.000000000 Z
+  title: !binary |-
+    U2ViYXN0aWFuIEZpdHplazogIkRlciBOYWNodHdhbmRsZXIi
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/346-Sebastian-Fitzek-Der-Nachtwandler.html
+- user: princess
+  date: 2013-06-17 16:21:17.000000000 Z
+  title: !binary |-
+    UC5DLiBDYXN0OiAiQXVzZXJzZWhlbiI=
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/345-P.C.-Cast-Ausersehen.html
+- user: princess
+  date: 2013-06-11 19:40:00.000000000 Z
+  title: !binary |-
+    RWR3YXJkIFNub3dkZW4gaXN0IGVpbiBIZWxk
+  url: http://blog.querulantin.de/serendipity/index.php?/archives/344-Edward-Snowden-ist-ein-Held.html

scripts/update-planetfeeds.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'yaml'
+require 'feedzirra'
+
+def getBlogroll(blogroll_file)
+  blogroll_raw = YAML.load_file(blogroll_file)
+  blogroll = { }
+  blogroll_raw['blogroll'].each { |blog| blogroll[blog['feed']]=blog['user'] }
+  blogroll
+end
+
+
+blogroll_file = ARGV[0]
+blogposts_file = ARGV[1]
+
+# Read existing data
+blogposts = if File.exists?(blogposts_file)
+              YAML.load_file(blogposts_file)
+            else
+              { }
+            end
+if !blogposts['blogposts']
+  blogposts['blogposts'] = []
+end
+blogroll = getBlogroll(blogroll_file)
+
+# Build list for detecting duplicates
+posturls = blogposts['blogposts'].map { |post| post['url'] }
+
+# Read feed
+feeds = Feedzirra::Feed.fetch_and_parse(blogroll.keys)
+
+# Add feed data
+feeds.each do |feed,data|
+  data.entries.each do |posting|
+    if !posturls.include?(posting.url)
+      postdata = { }
+      postdata['user'] = blogroll[feed]
+      postdata['date'] = posting.published
+      postdata['title'] = posting.title
+      postdata['url'] = posting.url
+      blogposts['blogposts'] << postdata
+    end
+  end
+end
+
+# Sort, limit list
+blogposts['blogposts'].sort! { |a,b| b['date'] <=> a['date'] }
+blogposts['blogposts'] = blogposts['blogposts'][0..19]
+
+# Output
+File.open(blogposts_file, 'w+') {|f| f.write(blogposts.to_yaml) }
+