Monthly Archives: August 2014

[UPDATE] Blockieren von XMLRPC Attacken

Seit ein paar Tagen bin ich in irgend einer Liste für XML-RPC Brute Force Attacken gelandet und die müllen meine limitierten Apache Slots zu. Lösung:

#!/bin/bash
# script: block xmlrpc attacks
# author: Steffen Wirth <s.wirth@itbert.de>
 
LOGFILE="/var/log/apache2/access.log"
LASTLINES="20"
MAXCOUNT="5"
 
LIST=$(tail -n$LASTLINES $LOGFILE |grep "xmlrpc.php" | awk '{print $1}' | sort -n | uniq -c)
 
if [ -n "$LIST" ]; then
        while read -r count ip ; do
                if [ $count -ge $MAXCOUNT ]; then
                        iptables -A INPUT -s $ip -j DROP
                        logger -t "XMLRPC" "blocked ip $ip"
                fi
        done <<< "$LIST"
fi

wie immer auch im gist.github.com

Wie Sebastian korrekt darauf hinwies, wenn man fail2ban installiert hat ist es viel einfacher:

# grep -v "^#" /etc/fail2ban/filter.d/apache-xmlrpc.conf 

[INCLUDES]

before = apache-common.conf

[Definition]

failregex = ^ .*POST .*xmlrpc\.php.*

ignoreregex = 
# grep apache-xmlrpc /etc/fail2ban/jail.conf -A3
[apache-xmlrpc]
enabled = true
port    = http,https
filter  = apache-xmlrpc
logpath = /var/log/apache*/*access.log
maxretry = 5

Soundcloud Favorite Downloader

Früher sammelte man noch MP3 Dateien wie wild, aber das ist längst Vergangenheit. Heutzutage kann man (oder ich) gemütlich neue und alte Musik auf Soundcloud oder ähnlichen hören. Ich höre eigentlich fast ausschließlich Musik auf Soundcloud. Es gibt einfach sehr gute Musik, gerade Remixe von Liedern auf der Platform.

Leider kann man die Lieder nicht in der Soundcloud App cachen bzw. Offline hören. Deswegen ist es für das Handy eigentlich gänzlich ungeeignet, wenn man nicht seinen Traffic dafür verbrauchen will. Zum Glück bieten manche Künstler ihre Lieder zum kostenlosen Download an und das kann man sich ja zunutze machen. Dafür habe ich ein kleines Script zusammen gebastelt, das die favorisierten Lieder herunterlädt.

#!/usr/bin/perl
# script: download favorite songs from your soundcloud stream
# author: Steffen Wirth <s.wirth@itbert.de>

use WebService::Soundcloud;
use XML::Simple;
no warnings 'utf8';

# create a new app -> soundcloud.com/you/apps/new
my $client_id = "YOUR_CLIEND_ID";
my $client_secret = "YOUR_CLIENT_SECRET";
# soundcloud username and password
my $username = "YOUR_USERNAME";
my $password = "YOUR_PASSWORD";
# download path
my $file_path = "/tmp/";
    
my $scloud = WebService::Soundcloud->new($client_id, $client_secret,
	{ username => $username, password => $password, response_format => 'xml' }
);
    
# get access token
my $access_token = $scloud->get_access_token();
my $oauth_token = $access_token->{access_token};

# get favorites tracks
my $followings = $scloud->get('/users/' . $username . '/favorites');

$xml = XML::Simple->new;
$xml = XMLin($followings->content);

foreach my $item (@{$xml->{track}}) {
	my $id = $item->{id}->{content};
	my $title = $item->{title};
	my $downloadable = $item->{downloadable}->{content};
	my $downloadurl = $item->{'download-url'};

	# only download songs that are downloadable
	if ($downloadable eq "true") {

		# download track
		$title =~ s/\ /_/g;
		my $dest_file = $file_path . $id . "_" . $title . ".mp3";

		unless (-e $dest_file) {
			print "DOWNLOAD: $title ($id)" . "\n"; 
			#my $path = $scloud->download($id, $file_path);

			# verrrrrrry ugly way, but $scloud->download(); is not working.
			# have fun with big files :)
			my $track = $scloud->get($downloadurl);
			my $sound = $track->content;
			open (TRACK, ">>$dest_file");
				print TRACK $sound;
			close(TRACK);

		} else {
			print "IGNORING TRACK $title ($id)" . "\n";
		}
	}
	
}

oder auf gist.github.com

Update: ABER ABER, was machen wir mit den Liedern die wir nicht offiziell herunterladen dürfen? Dafuer gibt es Dienste wie zum Beispiel anything2mp3.com. Damit kann man auch “diese” Lieder herunterladen. Wir sind Faul, also los:

#!/usr/bin/perl
# script: download favorite songs from your soundcloud stream
# author: Steffen Wirth 

use WebService::Soundcloud;
use XML::Simple;
use HTTP::Cookies;
use LWP::Simple;
use Encode qw(encode_utf8);
no warnings 'utf8';

### settings ###

# create a new app -> soundcloud.com/you/apps/new
my $client_id = "YOUR_CLIEND_ID";
my $client_secret = "YOUR_CLIENT_SECRET";
# soundcloud username and password
my $username = "YOUR_USERNAME";
my $password = "YOUR_PASSWORD";
# download path
my $file_path = "/tmp/";
# download service
my $url = "http://anything2mp3.com/de";

### script ###

my $cookie_jar = HTTP::Cookies->new( 
    file => 'lwp_cookies.txt',
    autosave => 1,
    ignore_discard => 1,
);

my $ua = LWP::UserAgent->new;
	$ua->agent("Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0");
	$ua->timeout(30);
	$ua->cookie_jar($cookie_jar);

my $scloud = WebService::Soundcloud->new($client_id, $client_secret,
	{ username => $username, password => $password, response_format => 'xml' }
);

# get access token
my $access_token = $scloud->get_access_token();
my $oauth_token = $access_token->{access_token};

# get favorites tracks
my $followings = $scloud->get('/users/' . $username . '/favorites');

$xml = XML::Simple->new;
$xml = XMLin($followings->content);

foreach my $item (@{$xml->{track}}) {
	my $id = $item->{id}->{content};
	my $title = $item->{title};
	my $downloadable = $item->{downloadable}->{content};
	my $downloadurl = $item->{'download-url'};
	my $permalink = $item->{'permalink-url'};

	# title and file 
	$title =~ s/\ /_/g;
	my $dest_file = $file_path . $id . "_" . $title . ".mp3";
	$dest_file = encode_utf8($dest_file);

	# only download songs that are downloadable
	if ($downloadable eq "true") {

		unless (-e $dest_file) {
			print "DOWNLOAD: $title" . "\n"; 
			#my $path = $scloud->download($id, $file_path);

			# verrrrrrry ugly way, but $scloud->download(); is not working.
			# have fun with big files :)
			my $track = $scloud->get($downloadurl);
			my $sound = $track->content;
			open (TRACK, ">>$dest_file");
				print TRACK $sound;
			close(TRACK);

		} else {
			print "IGNORING TRACK $title" . "\n";
		}
	} else {
		# lets download tracks that we are not supposed to

		unless (-e $dest_file) {
			print "D0WNL04D $title" . "\n";
			my $request = $ua->post($url,
				{
					url => $permalink,
					op => 'Convert',
					form_build_id => 'form-iKcdS_GJM5mRuRicuFJKS7wGB8oR7zbY6YbVeV4cjtM', # not sure how long this is valid
					form_id => 'videoconverter_form',
			});

			my $html = $ua->get($url . '/kostenlose-online-soundcloud-youtube-mp3-converter');
			$html = $html->content;

			while ($html =~ m@(((http://anything2mp3.com/de/system/temporary/mp3/))\S+[^.,!? ])@g) {
				$dl = $1;
				$dl =~ s/\>\Click//g;
			}

			($file) = $dl =~ m!([^/]+)$!;
			$file =~ s/\?download=1//g;

			my $track = $ua->get($dl);
			my $sound = $track->content;
			open (TRACK, ">>$dest_file");
				print TRACK $sound;
			close(TRACK);
		} else {
			print "IGNORING TRACK $title" . "\n";
		}

	}
	
}

auch auf gist.github.com