fix: disallow crawling from /_astro/ (#399)

* fix: disallow crawling from `/_astro/`

* fix: add missing newline at end of robots.txt.ts file

---------

Co-authored-by: Joey Chen <JoeyC-Dev@users.noreply.github.com>
Co-authored-by: L4Ph <4ranci0ne@gmail.com>

src/pages/robots.txt.ts

@@ -2,7 +2,7 @@ import type { APIRoute } from "astro";
 
 const robotsTxt = `
 User-agent: *
-Allow: /
+Disallow: /_astro/
 
 Sitemap: ${new URL("sitemap-index.xml", import.meta.env.SITE).href}
 `.trim();