Ack, there were some regex problems in the robots.txt parser, the attached
version should work properly, even though it does resort to a more
brute-force approach.
last response to myself for the day, promise ;)
-HD
##
# This plugin was written by H D Moore <[EMAIL PROTECTED]>
##
if(description)
{
# script_id(????);
script_version ("$Revision: 1.2 $");
name["english"] = "Directory Scanner";
script_name(english:name["english"]);
desc["english"] = "None Yet";
script_description(english:desc["english"]);
summary["english"] = "Directory Scanner";
script_summary(english:summary["english"]);
script_category(ACT_GATHER_INFO);
script_copyright(english:"This script is Copyright (C) 2002 Digital Defense
Inc.");
family["english"] = "Misc.";
script_family(english:family["english"]);
script_dependencie("find_service.nes");
script_require_ports(80);
exit(0);
}
function check_req (port, url)
{
soc = open_sock_tcp(port);
if (!soc) return (0);
req = http_get(item:url, port:port);
send(socket:soc, data:req);
http_resp = recv(socket:soc, length:16384);
close (soc);
return(string(http_resp));
}
function check_dir_list (dir)
{
for (CDC=0; dirs[CDC]; CDC=CDC+1)
{
if (dirs[CDC] == dir)
{
return(1);
}
}
return(0);
}
dirs[0] = ".cobalt";
dirs[1] = "1";
dirs[2] = "10";
dirs[3] = "2";
dirs[4] = "3";
dirs[5] = "4";
dirs[6] = "5";
dirs[7] = "6";
dirs[8] = "7";
dirs[9] = "8";
dirs[10] = "9";
dirs[11] = "AdminWeb";
dirs[12] = "Admin_files";
dirs[13] = "Administration";
dirs[14] = "CVS";
dirs[15] = "DMR";
dirs[16] = "HB";
dirs[17] = "HBTemplates";
dirs[18] = "Images";
dirs[19] = "Log";
dirs[20] = "Mail";
dirs[21] = "PDG_Cart";
dirs[22] = "Stats";
dirs[23] = "StoreDB";
dirs[24] = "Templates";
dirs[25] = "WebShop";
dirs[26] = "WebTrend";
dirs[27] = "Web_store";
dirs[28] = "_backup";
dirs[29] = "_notes";
dirs[30] = "_old";
dirs[31] = "_private";
dirs[32] = "a";
dirs[33] = "access";
dirs[34] = "account";
dirs[35] = "accounting";
dirs[36] = "adm";
dirs[37] = "admin";
dirs[38] = "admin-bak";
dirs[39] = "admin-old";
dirs[40] = "admin.back";
dirs[41] = "administration";
dirs[42] = "administrator";
dirs[43] = "adminweb";
dirs[44] = "analog";
dirs[45] = "app";
dirs[46] = "applets";
dirs[47] = "apps";
dirs[48] = "archive";
dirs[49] = "archives";
dirs[50] = "asp";
dirs[51] = "atc";
dirs[52] = "b";
dirs[53] = "back";
dirs[54] = "backend";
dirs[55] = "backup";
dirs[56] = "bak";
dirs[57] = "bank";
dirs[58] = "banner";
dirs[59] = "beta";
dirs[60] = "billpay";
dirs[61] = "bin";
dirs[62] = "buy";
dirs[63] = "buynow";
dirs[64] = "c";
dirs[65] = "cache-stats";
dirs[66] = "cart";
dirs[67] = "ccard";
dirs[68] = "ccards";
dirs[69] = "cgi";
dirs[70] = "cgi-bin";
dirs[71] = "cgi-csc";
dirs[72] = "cgi-local";
dirs[73] = "cgi-win";
dirs[74] = "cgis";
dirs[75] = "config";
dirs[76] = "core";
dirs[77] = "counter";
dirs[78] = "credit";
dirs[79] = "cron";
dirs[80] = "crons";
dirs[81] = "css";
dirs[82] = "customers";
dirs[83] = "cvsweb";
dirs[84] = "d";
dirs[85] = "dat";
dirs[86] = "data";
dirs[87] = "database";
dirs[88] = "db";
dirs[89] = "dbase";
dirs[90] = "design";
dirs[91] = "dev";
dirs[92] = "devel";
dirs[93] = "development";
dirs[94] = "dl";
dirs[95] = "doc";
dirs[96] = "doc-html";
dirs[97] = "docs";
dirs[98] = "down";
dirs[99] = "download";
dirs[100] = "downloads";
dirs[101] = "dump";
dirs[102] = "e";
dirs[103] = "eforum";
dirs[104] = "email";
dirs[105] = "emailclass";
dirs[106] = "employees";
dirs[107] = "empoyees";
dirs[108] = "error";
dirs[109] = "estmt";
dirs[110] = "etc";
dirs[111] = "exc";
dirs[112] = "exchange";
dirs[113] = "exe";
dirs[114] = "f";
dirs[115] = "file";
dirs[116] = "files";
dirs[117] = "forms";
dirs[118] = "forum";
dirs[119] = "fpadmin";
dirs[120] = "ftp";
dirs[121] = "g";
dirs[122] = "guestbook";
dirs[123] = "guests";
dirs[124] = "hidden";
dirs[125] = "hide";
dirs[126] = "hit_tracker";
dirs[127] = "hitmatic";
dirs[128] = "home";
dirs[129] = "htbin";
dirs[130] = "htdocs";
dirs[131] = "html";
dirs[132] = "ibank";
dirs[133] = "ibill";
dirs[134] = "icons";
dirs[135] = "idea";
dirs[136] = "ideas";
dirs[137] = "imagery";
dirs[138] = "img";
dirs[139] = "imp";
dirs[140] = "import";
dirs[141] = "inc";
dirs[142] = "include";
dirs[143] = "includes";
dirs[144] = "incoming";
dirs[145] = "info";
dirs[146] = "install";
dirs[147] = "intranet";
dirs[148] = "java";
dirs[149] = "jave";
dirs[150] = "jdbc";
dirs[151] = "js";
dirs[152] = "lib";
dirs[153] = "libraries";
dirs[154] = "library";
dirs[155] = "links";
dirs[156] = "loader";
dirs[157] = "log";
dirs[158] = "logfile";
dirs[159] = "logfiles";
dirs[160] = "logger";
dirs[161] = "logging";
dirs[162] = "login";
dirs[163] = "logon";
dirs[164] = "logs";
dirs[165] = "mail";
dirs[166] = "mall_log_files";
dirs[167] = "manual";
dirs[168] = "marketing";
dirs[169] = "msql";
dirs[170] = "mysql";
dirs[171] = "new";
dirs[172] = "odbc";
dirs[173] = "old";
dirs[174] = "oracle";
dirs[175] = "order";
dirs[176] = "orders";
dirs[177] = "outgoing";
dirs[178] = "pages";
dirs[179] = "passport";
dirs[180] = "password";
dirs[181] = "passwords";
dirs[182] = "perl";
dirs[183] = "perl5";
dirs[184] = "phorum";
dirs[185] = "php";
dirs[186] = "phpMyAdmin";
dirs[187] = "poll";
dirs[188] = "polls";
dirs[189] = "postgres";
dirs[190] = "priv";
dirs[191] = "private";
dirs[192] = "prv";
dirs[193] = "pub";
dirs[194] = "public";
dirs[195] = "purchase";
dirs[196] = "purchases";
dirs[197] = "pw";
dirs[198] = "rdp";
dirs[199] = "register";
dirs[200] = "registered";
dirs[201] = "reports";
dirs[202] = "reseller";
dirs[203] = "restricted";
dirs[204] = "retail";
dirs[205] = "root";
dirs[206] = "sales";
dirs[207] = "script";
dirs[208] = "scripts";
dirs[209] = "search";
dirs[210] = "secret";
dirs[211] = "sell";
dirs[212] = "setup";
dirs[213] = "shop";
dirs[214] = "shopper";
dirs[215] = "site";
dirs[216] = "software";
dirs[217] = "source";
dirs[218] = "sql";
dirs[219] = "staff";
dirs[220] = "stat";
dirs[221] = "statistic";
dirs[222] = "statistics";
dirs[223] = "stats";
dirs[224] = "status";
dirs[225] = "store";
dirs[226] = "support";
dirs[227] = "sys";
dirs[228] = "system";
dirs[229] = "tech";
dirs[230] = "temp";
dirs[231] = "templates";
dirs[232] = "test";
dirs[233] = "test-cgi";
dirs[234] = "testing";
dirs[235] = "tmp";
dirs[236] = "tools";
dirs[237] = "trafficlog";
dirs[238] = "tree";
dirs[239] = "updates";
dirs[240] = "usage";
dirs[241] = "user";
dirs[242] = "users";
dirs[243] = "usr";
dirs[244] = "ustats";
dirs[245] = "vfs";
dirs[246] = "web";
dirs[247] = "web800fo";
dirs[248] = "webadmin";
dirs[249] = "webboard";
dirs[250] = "webcart";
dirs[251] = "webcart-lite";
dirs[252] = "webdata";
dirs[253] = "webimages";
dirs[254] = "webimages2";
dirs[255] = "weblog";
dirs[256] = "weblogs";
dirs[257] = "website";
dirs[258] = "webstats";
dirs[259] = "wstats";
dirs[260] = "wusage";
dirs[261] = "www";
dirs[262] = "www-sql";
dirs[263] = "wwwjoin";
dirs[264] = "wwwlog";
dirs[265] = "wwwstats";
dirs[266] = "zipfiles";
# this needs to be updated to match the above list
dirs_last = 266;
# these are the strings used by the 404 checks
errmsg[0] = "not found";
errmsg[1] = "404";
errmsg[2] = "error has occurred";
errmsg[3] = "firewall-1 message";
errmsg[4] = "Reload acp_userinfo database";
errmsg[5] = "IMail Server Web Messaging";
errmsg[6] = "HP Web JetAdmin";
errmsg[7] = "Error processing SSI file";
errmsg[8] = "ExtendNet DX Configuration";
errmsg[9] = "not available";
errmsg[10] = "Content-Length: 0";
debug = 1;
if(debug) display("\n::[ DDI Directory Scanner running in debug mode\n::\n");
report = string("The following directories were discovered: ");
found = 0;
authreport = string("The following directories require authentication: ");
authfound = 0;
fake404 = string("");
Check200 = 1;
Check401 = 1;
Check403 = 1;
port = get_kb_item("Services/www");
if(!port)port = 80;
if(!get_port_state(port))
{
if(debug) display(":: Error: port ", port, " was not open on target.\n");
exit(0);
}
##
# pull the robots.txt file
##
soc = open_sock_tcp(port);
if (!soc)
{
if(debug) display(":: Error: could not connect to port ", port, " on target.\n");
exit(0);
}
display(":: Checking for robots.txt...\n");
req = http_get(item:"/robots.txt", port:port);
send(socket:soc, data:req);
http_line = recv_line(socket:soc, length:16384);
if (ereg(pattern:"HTTP/1.[01] 200", string:http_line))
{
while (http_line = recv_line(socket:soc, length:16384))
{
if (ereg(pattern:"disallow:", string:http_line, icase:TRUE))
{
# yes, i suck at regex's in nasl. I want my \s+!
robot_dir = ereg_replace(pattern:"disallow:\W*(/.*)\W$", string:http_line,
replace:"\1", icase:TRUE);
robot_dir = ereg_replace(pattern:"/(.*)(\W*)$", string:robot_dir,
replace:"\1", icase:TRUE);
robot_dir = ereg_replace(pattern:"/$|\?$", string:robot_dir, replace:"",
icase:TRUE);
if (debug) display(":: Discovered directory through robots.txt: '",
robot_dir, "'\n");
if (! check_dir_list(dir:robot_dir))
{
# add directory to the list
dirs_last = dirs_last + 1;
dirs[dirs_last] = robot_dir;
if (debug) display(":: Directory '", robot_dir, "' added to test
list\n");
} else {
if (debug) display(":: Directory '", robot_dir, "' already exists in
test list\n");
}
}
}
}
close(soc);
##
# test for servers which return 200/403/401 for everything
##
soc = open_sock_tcp(port);
if (!soc)
{
if(debug) display(":: Error: could not connect to port ", port, " on target.\n");
exit(0);
}
display(":: Checking for non-standard server responses\n");
req = http_get(item:"/NonExistent/", port:port);
send(socket:soc, data:req);
http_resp = recv(socket:soc, length:16384);
close (soc);
if(ereg(pattern:"HTTP/1.[01] 200", string: http_resp))
{
fake404 = 0;
if(debug) display(":: This server returns 200 for non-existent directories.\n");
for(i=0;errmsg[i];i=i+1)
{
if (ereg(pattern:errmsg[i], string:http_resp, icase:TRUE))
{
fake404 = errmsg[i];
if(debug) display(":: Using '", fake404, "' as an indication of a 404
error\n");
}
}
if (!fake404)
{
if(debug) display(":: Could not find an error string to match against for the
fake 404 response.\n");
if(debug) display(":: Checks which rely on 200 responses are being
disabled\n");
Check200 = 0;
}
} else {
fake404 = string("BadString0987654321*DDI*");
}
if(ereg(pattern:"HTTP/1.[01] 401", string: http_resp))
{
if(debug) display(":: This server requires authentication for non-existent
directories, disabling 401 checks.\n");
Check401 = 0;
}
if(ereg(pattern:"HTTP/1.[01] 403", string: http_resp))
{
if(debug) display(":: This server returns a 403 for non-existent directories,
disabling 403 checks.\n");
Check403 = 0;
}
display(":: Starting the directory scan...\n");
for(i=0;dirs[i];i=i+1)
{
res = check_req(port:port, url:string("/", dirs[i], "/"));
if( Check200 &&
ereg(pattern:"HTTP/1.[01] 200", string:res) &&
! (ereg(pattern:fake404, string:res, icase:TRUE))
)
{
if(debug) display(":: Discovered: " , dirs[i], "\n");
if(found)
{
report = report + ", /" + dirs[i];
found=found+1;
} else {
report = report + "/" + dirs[i];
}
found=found+1;
}
if(Check403 && ereg(pattern:"HTTP/1.[01] 403", string: res))
{
if (debug) display(":: Got a 403 for ", dirs[i], ", checking for file in the
directory...\n");
res2 = check_req(port:port, url:string("/", dirs[i], "/NonExistent.html"));
if(ereg(pattern:"HTTP/1.[01] 403", string:res2))
{
# the whole directory appears to be protected
if (debug) display(":: 403 applies to the entire directory \n");
} else {
if (debug) display(":: 403 applies to just directory indexes \n");
# the directory just has indexes turned off
if(debug) display(":: Discovered: " , dirs[i], "\n");
if(found)
{
report = report + ", /" + dirs[i];
} else {
report = report + "/" + dirs[i];
}
found=found+1;
}
}
if(Check401 && ereg(pattern:"HTTP/1.[01] 401", string: res))
{
if (debug) display(":: Got a 401 for ", dirs[i], "\n");
if(found)
{
authreport = authreport + ", /" + dirs[i];
} else {
authreport = authreport + "/" + dirs[i];
}
authfound=authfound+1;
}
}
result = string("");
if (found)
{
result = report;
}
if (authfound)
{
result = result + string("\n", authreport);
}
if (strlen(result))
{
security_note(port:port, data:result);
}
