grep robots.txt access_log | wc -l sort agent_log | cut -f2- -f']' | uniq > agent_list order allow,deny allow from all deny from 192.168.1.
