[squid-users] Caching youtube content again

Hello list readers,

I'm trying to find a stable and reliable solution to cache youtube (amongs other dynamic stuff suppliers) content. Three day ago, I did manage to cache some stuff, but it was not really reliable. It would work 1 time on 2. I had no particular configuration, I only used the wiki squid cache directives.

Now, I've been digging the list and googled a lot about the subject, but I cannot really find something stable.

I'm using squid 2.7stable 6 on an opensuse 11.2.
I've also been doing tests with Lucas Diaz's rewriter & configuration.

Thanks for tips & hints people will provide :)

Thanks,
Guillaume

As now, the configuration is as follow :
------------------------------------------------

HELPER :
-------------

#!/usr/bin/perl
$|=1;
while (<>) {
@X = split;
$url = $X[0];
if ($url=~s@^http://(.*?)/videoplayback\?(.*)id=(.*?)&.*@squid://videos.youtube.INTERNAL/ID=$3@){}
elsif
    ($url =~s@^http://(.*?)/videoplayback\?(.*)id=(.*?)@squid://videos.youtube.INTERNAL/ID=$3@){}
elsif
    ($url =~s@^http://(.*?)/videoplay\?(.*)id=(.*?)&.*@squid://videos.youtube.INTERNAL/ID=$3@){}
elsif
    ($url =~s@^http://(.*?)/videoplay\?(.*)id=(.*?)@squid://videos.youtube.INTERNAL/ID=$3@){}
elsif
    ($url =~s@^http://(.*?)/get_video\?(.*)video_id=(.*?)&.*@squid://videos.youtube.INTERNAL/ID=$3@){}
elsif
    ($url =~s@^http://(.*?)/get_video\?(.*)video_id=(.*?)@squid://videos.youtube.INTERNAL/ID=$3@){}
elsif
    ($url =~s@^http://(.*?)rapidshare(.*?)/files/(.*?)/(.*?)/(.*?)@squid://files.rapidshare.INTERNAL/$5@){}
elsif
    ($url =~s@^http://(.*?)fbcdn\.net/(.*?)/(.*?)/(.*?\.jpg)@squid://files.facebook.INTERNAL/$4@){}
elsif
    ($url =~s@^http://contenidos2(.*?)/(.*?)@squid://files.contenidos2.INTERNAL/$2@){}
elsif
    ($url =~s@^http://cdn(.*?)/([0-9a-zA-Z_-]*?\.flv)@squid://files.cdn.INTERNAL/$2@){}
elsif
    ($url =~s@^http://web.vxv.com/data/media/(.*?)@squid://files.vxv.INTERNAL/$1@){}
elsif
    ($url =~s@^http://(.*?)megaupload\.com/files/(.*?)/(.*?)@squid://files.megaupload.INTERNAL/$3@){}
elsif
    ($url =~s@^http://(.*?)mediafire\.com/(.*?)/(.*?)@squid://files.megaupload.INTERNAL/$3@){}
elsif
    ($url =~s@^http://(.*?)depositfiles\.com/(.*?)/(.*?)/(.*?)@squid://files.megaupload.INTERNAL/$4@){}
elsif
    ($url =~s@^http://(.*?)\.files\.youporn\.com\/(.*?)\/([0-9a-zA-Z_-]*?\.flv)\?.*@squid://videos.youporn.INTERNAL/$3@){}
elsif
    ($url =~s@^http://(.*?)\.files\.youporn\.com\/(.*?)\/([0-9a-zA-Z_-]*?\.flv)@squid://videos.youporn.INTERNAL/$3@){}
elsif
    ($url =~s@^http://(.*?)\.tube8\.com\/(.*?)\/([0-9a-zA-Z_-]*?\.flv)\?.*@squid://videos.tube8.INTERNAL/$3@){}
elsif
    ($url =~s@^http://(.*?)\.tube8\.com\/(.*?)\/([0-9a-zA-Z_-]*?\.flv)@squid://videos.tube8.INTERNAL/$3@){}
elsif
    ($url =~s@^http://(.*?)megaporn\.com\/files\/(.*?)\/(.*?)@squid://files.megaporn.INTERNAL/$3@){}
print "$url\n"; }

SQUID.CONF :
-------------------

acl all src all
acl manager proto cache_object
acl localhost src 127.0.0.1/32
acl to_localhost dst 127.0.0.0/8
acl localnet src 10.0.0.0/8     # RFC1918 possible internal network
acl localnet src 172.16.0.0/12  # RFC1918 possible internal network
acl localnet src 192.168.0.0/16 # RFC1918 possible internal network
acl SSL_ports port 443
acl Safe_ports port 80          # http
acl Safe_ports port 21          # ftp
acl Safe_ports port 443         # https
acl Safe_ports port 70          # gopher
acl Safe_ports port 210         # wais
acl Safe_ports port 1025-65535  # unregistered ports
acl Safe_ports port 280         # http-mgmt
acl Safe_ports port 488         # gss-http
acl Safe_ports port 591         # filemaker
acl Safe_ports port 777         # multiling http
acl CONNECT method CONNECT
http_access allow manager localhost
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access allow localnet
http_access allow localhost
http_access deny all
icp_access allow localnet
icp_access deny all
http_port 3128
cache_mem 20 MB
maximum_object_size_in_memory 2 MB
memory_replacement_policy lru
cache_replacement_policy lru
cache_dir aufs /var/cache/squid 1000 16 256
minimum_object_size -1 KB
maximum_object_size 88192 KB
logformat squid %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
logformat squidmime %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt [%>h] [%<h]
logformat common %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st %Ss:%Sh
logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %Hs %<st "%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh
access_log /var/log/squid/access.log squid
cache_log /var/log/squid/cache.log
cache_store_log /var/log/squid/store.log
acl store_rewrite_list url_regex -i \.youtube\.com\/get_video\?
acl store_rewrite_list url_regex -i \.youtube\.com\/watch\?
acl store_rewrite_list url_regex -i \.youtube\.com\/videoplayback\.youtube\.com\/videoplay \.youtube\.com\/get_video\?
acl store_rewrite_list url_regex -i \.youtube\.[a-z][a-z]\/videoplayback\.youtube\.[a-z][a-z]\/videoplay \.youtube\.[a-z][a-z]\/get_video\?
acl store_rewrite_list url_regex -i \.googlevideo\.com\/videoplayback\.googlevideo\.com\/videoplay \.googlevideo\.com\/get_video\?
acl store_rewrite_list url_regex -i \.google\.com\/videoplayback\.google\.com\/videoplay \.google\.com\/get_video\?
acl store_rewrite_list url_regex -i \.google\.[a-z][a-z]\/videoplayback\.google\.[a-z][a-z]\/videoplay \.google\.[a-z][a-z]\/get_video\?
acl store_rewrite_list url_regex -i (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(
25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/videoplayback\?
acl store_rewrite_list url_regex -i (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(
25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/videoplay\?
acl store_rewrite_list url_regex -i (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(
25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/get_video\?
acl store_rewrite_list url_regex -i (25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(
25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/watch\?
acl QUERY urlpath_regex cgi-bin \?
storeurl_access allow store_rewrite_list
storeurl_access deny all
storeurl_rewrite_program /etc/squid/store_url_rewrite
storeurl_rewrite_children 15
cache allow store_rewrite_list
cache allow all # ADDITION : IF COMMENTED, SQUID WILL JUST CACHE NOTHING. NOR YOUTUBE, NOR ANYTHING ELSE.
cache deny QUERY
refresh_pattern ^http://(.*?)/watch\? 10080 90% 999999 override-expire ignore-no-cache ignore-private
refresh_pattern ^http://(.*?)/get_video\? 10080 90% 999999 override-expire ignore-no-cache ignore-private
refresh_pattern ^http://(.*?)/videodownload\? 10080 90% 999999 override-expire ignore-no-cache ignore-private
refresh_pattern ^http://i(.*?).photobucket.com/albums/(.*?)/(.*?)/(.*?)\? 43200 90% 999999 override-expire ignore-no-cache ignore-private
refresh_pattern ^http://vid(.*?).photobucket.com/albums/(.*?)/(.*?)\? 43200 90% 999999 override-expire ignore-no-cache ignore-private
refresh_pattern ^ftp:           1440    20%     10080
refresh_pattern ^gopher:        1440    0%      1440
refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
refresh_pattern .               0       20%     4320
quick_abort_min 0
quick_abort_max 0
quick_abort_pct 98
acl shoutcast rep_header X-HTTP09-First-Line ^ICY.[0-9]
upgrade_http0.9 deny shoutcast
acl apache rep_header Server ^Apache
broken_vary_encoding allow apache
error_directory /usr/share/squid/errors/English
coredump_dir /var/cache/squid

                                               
_________________________________________________________________
Hotmail: Powerful Free email with security by Microsoft.
https://signup.live.com/signup.aspx?id=60969
Received on Wed Mar 03 2010 - 10:11:45 MST