[
 {
  "id": "2096838373524812149",
  "text": "The cost problem in web scraping for AI agents is not network latency. It is LLM inference on every fetch.\n\nA naive agent that scrapes a product catalog runs an LLM over the raw HTML for every page it fetches to extract structured data. If the catalog has ten thousand products and the LLM call costs half a cent per page, the extraction run costs fifty dollars. Run it daily and the annual cost is eighteen thousand dollars for one data source.\n\nSelf-healing CSS selectors change the economics. The first run uses an LLM to identify the right selectors for each field. Subsequent runs use the selectors directly and only trigger LLM inference again when a selector breaks because the page structure changed. For a stable site, the LLM runs once. For a site that redesigns quarterly, it runs four times a year per field.\n\nThe MCP server layer matters because it means the scraper is callable as a tool from inside an agent loop, not just as a standalone script. An agent that needs price data can call DeepScrape as a tool, get structured output back, and continue without leaving the agentic context.\n\nFirecrawl, Apify, and Browserless each solve a piece of this. None of them combine self-healing selectors with an MCP interface in a single open-source package you can run yourself.\n\nhttps://t.co/JZYTWY8roq\n\nhttps://t.co/tIrnvH5vb2",
  "retweetCount": 2,
  "replyCount": 6,
  "likeCount": 5,
  "quoteCount": 0,
  "viewCount": 433,
  "bookmarkCount": 1,
  "card": {
   "bindingValues": {
    "photoImageFullSizeLarge": {
     "height": 419,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=800x419",
     "width": 800
    },
    "thumbnailImage": {
     "height": 144,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=144x144",
     "width": 144
    },
    "description": "Contribute to stretchcloud/deepscrape development by creating an account on GitHub.",
    "domain": "github.com",
    "thumbnailImageLarge": {
     "height": 320,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=800x320_1",
     "width": 320
    },
    "summaryPhotoImageSmall": {
     "height": 202,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=386x202",
     "width": 386
    },
    "thumbnailImageOriginal": {
     "height": 2048,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=orig",
     "width": 2048
    },
    "site": {
     "id": "13334762"
    },
    "photoImageFullSizeSmall": {
     "height": 202,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=386x202",
     "width": 386
    },
    "summaryPhotoImageLarge": {
     "height": 419,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=800x419",
     "width": 800
    },
    "thumbnailImageSmall": {
     "height": 100,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=100x100",
     "width": 100
    },
    "thumbnailImageXLarge": {
     "height": 2048,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=png&name=2048x2048_2_exp",
     "width": 2048
    },
    "photoImageFullSizeOriginal": {
     "height": 2048,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=orig",
     "width": 2048
    },
    "photoImageFullSizeAltText": "Contribute to stretchcloud/deepscrape development by creating an account on GitHub.",
    "vanityUrl": "github.com",
    "photoImageFullSize": {
     "height": 314,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=600x314",
     "width": 600
    },
    "summaryPhotoImageAltText": "Contribute to stretchcloud/deepscrape development by creating an account on GitHub.",
    "thumbnailImageColor": {
     "palette": [
      {
       "percentage": 97.97,
       "rgb": {
        "blue": 255,
        "green": 255,
        "red": 255
       }
      },
      {
       "percentage": 1.17,
       "rgb": {
        "blue": 132,
        "green": 119,
        "red": 117
       }
      },
      {
       "percentage": 0.77,
       "rgb": {
        "blue": 199,
        "green": 126,
        "red": 38
       }
      }
     ]
    },
    "title": "GitHub - stretchcloud/deepscrape",
    "summaryPhotoImageColor": {
     "palette": [
      {
       "percentage": 97.97,
       "rgb": {
        "blue": 255,
        "green": 255,
        "red": 255
       }
      },
      {
       "percentage": 1.17,
       "rgb": {
        "blue": 132,
        "green": 119,
        "red": 117
       }
      },
      {
       "percentage": 0.77,
       "rgb": {
        "blue": 199,
        "green": 126,
        "red": 38
       }
      }
     ]
    },
    "summaryPhotoImageXLarge": {
     "height": 2048,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=png&name=2048x2048_2_exp",
     "width": 2048
    },
    "summaryPhotoImage": {
     "height": 314,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=600x314",
     "width": 600
    },
    "photoImageFullSizeColor": {
     "palette": [
      {
       "percentage": 97.97,
       "rgb": {
        "blue": 255,
        "green": 255,
        "red": 255
       }
      },
      {
       "percentage": 1.17,
       "rgb": {
        "blue": 132,
        "green": 119,
        "red": 117
       }
      },
      {
       "percentage": 0.77,
       "rgb": {
        "blue": 199,
        "green": 126,
        "red": 38
       }
      }
     ]
    },
    "photoImageFullSizeXLarge": {
     "height": 2048,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=png&name=2048x2048_2_exp",
     "width": 2048
    },
    "cardUrl": "https://t.co/JZYTWY8roq",
    "summaryPhotoImageOriginal": {
     "height": 2048,
     "url": "https://pbs.twimg.com/card_img/2101216242510200832/f76Wgfju?format=jpg&name=orig",
     "width": 2048
    }
   },
   "id": "https://t.co/JZYTWY8roq",
   "name": "summary_large_image",
   "platform": {
    "platform": {
     "audience": {
      "name": "production"
     },
     "device": {
      "name": "Swift",
      "version": "12"
     }
    }
   },
   "url": "https://t.co/JZYTWY8roq",
   "userReferences": [
    {
     "id": "13334762",
     "userName": "github",
     "name": "GitHub",
     "profilePicture": "https://pbs.twimg.com/profile_images/2051404708766507008/ATxxTJXO_normal.jpg",
     "isVerified": true,
     "profileBannerUrl": "https://pbs.twimg.com/profile_banners/13334762/1780668208",
     "isBlueVerified": true,
     "followers": 2710845,
     "following": 334,
     "favouritesCount": 8978,
     "statusesCount": 10908,
     "mediaCount": 3126,
     "verified": false,
     "description": "The AI-powered developer platform to build, scale, and deliver secure software.",
     "location": "San Francisco, CA",
     "createdAt": "Mon Feb 11 04:41:50 +0000 2008",
     "coverPicture": "https://pbs.twimg.com/profile_banners/13334762/1780668208",
     "profileBio": {
      "description": "The AI-powered developer platform to build, scale, and deliver secure software.",
      "entities": {
       "description": {},
       "url": {
        "urls": [
         {
          "displayUrl": "github.com",
          "expandedUrl": "http://github.com",
          "indices": [
           0,
           23
          ],
          "url": "https://t.co/bbJgfyzKzp"
         }
        ]
       }
      }
     },
     "url": "http://github.com",
     "verifiedType": "Business",
     "pinnedTweetIds": [
      "2098065317083906156"
     ],
     "possiblySensitive": false,
     "hasGraduatedAccess": true,
     "parodyCommentaryFanLabel": "None",
     "profileDescriptionLanguage": "en",
     "profileImageShape": "Square",
     "profileInterstitialType": "",
     "profileTranslatorType": "none",
     "superFollowEligible": false,
     "protected": false
    }
   ]
  },
  "conversationId": "2096838373524812149",
  "displayTextRange": [
   0,
   277
  ],
  "edit": {
   "editableUntilMsecs": "1788763800000",
   "editTweetIds": [
    "2096838373524812149"
   ]
  },
  "entities": {
   "hashtags": [],
   "urls": [],
   "userMentions": []
  },
  "isNoteTweet": true,
  "isQuoteStatus": true,
  "isReply": false,
  "isRetweet": false,
  "isTranslatable": false,
  "lang": "en",
  "noteTweet": {
   "entities": {
    "hashtags": [],
    "urls": [
     {
      "displayUrl": "github.com/stretchcloud/d…",
      "expandedUrl": "https://github.com/stretchcloud/deepscrape",
      "indices": [
       1286,
       1309
      ],
      "url": "https://t.co/JZYTWY8roq"
     },
     {
      "displayUrl": "x.com/jp54362/status…",
      "expandedUrl": "https://x.com/jp54362/status/2093291401496932832",
      "indices": [
       1311,
       1334
      ],
      "url": "https://t.co/tIrnvH5vb2"
     }
    ],
    "userMentions": []
   },
   "id": "Tm90ZVR3ZWV0OjIwOTY4MzgzNzMyODE1NzQ5MTI=",
   "isExpandable": true,
   "text": "The cost problem in web scraping for AI agents is not network latency. It is LLM inference on every fetch.\n\nA naive agent that scrapes a product catalog runs an LLM over the raw HTML for every page it fetches to extract structured data. If the catalog has ten thousand products and the LLM call costs half a cent per page, the extraction run costs fifty dollars. Run it daily and the annual cost is eighteen thousand dollars for one data source.\n\nSelf-healing CSS selectors change the economics. The first run uses an LLM to identify the right selectors for each field. Subsequent runs use the selectors directly and only trigger LLM inference again when a selector breaks because the page structure changed. For a stable site, the LLM runs once. For a site that redesigns quarterly, it runs four times a year per field.\n\nThe MCP server layer matters because it means the scraper is callable as a tool from inside an agent loop, not just as a standalone script. An agent that needs price data can call DeepScrape as a tool, get structured output back, and continue without leaving the agentic context.\n\nFirecrawl, Apify, and Browserless each solve a piece of this. None of them combine self-healing selectors with an MCP interface in a single open-source package you can run yourself.\n\nhttps://t.co/JZYTWY8roq\n\nhttps://t.co/tIrnvH5vb2"
  },
  "quotedTweetId": "2093291401496932832",
  "source": "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
  "type": "tweet",
  "url": "https://x.com/stretchcloud/status/2096838373524812149",
  "viewState": "EnabledWithCount",
  "createdAt": "Mon Sep 07 05:50:00 +0000 2026",
  "author": {
   "id": "266259493",
   "username": "stretchcloud",
   "name": "Prasenjit Sarkar",
   "followers": 2989,
   "following": 972,
   "verified": true,
   "isBlueVerified": true,
   "isVerified": true,
   "profilePicture": "https://pbs.twimg.com/profile_images/1966979113316409344/BkqBhdQb_normal.jpg",
   "coverPicture": "https://pbs.twimg.com/profile_banners/266259493/1466546448",
   "profileBannerUrl": "https://pbs.twimg.com/profile_banners/266259493/1466546448",
   "description": "Founder & Builder | Building https://t.co/ag2bhjo5r8, https://t.co/9EO3BU4jCx, https://t.co/vqTghjn5bf | AI Agents| 15x Patents | 7x Author | Building for Growth",
   "location": "London, England",
   "createdAt": "Mon Mar 14 23:22:17 +0000 2011",
   "statusesCount": 13292,
   "mediaCount": 1097,
   "protected": false,
   "url": "https://the-campfire.dev/",
   "favouritesCount": 3123,
   "isTranslator": false,
   "possiblySensitive": false,
   "verifiedType": "Blue",
   "profileBio": {
    "description": "Founder & Builder | Building https://t.co/ag2bhjo5r8, https://t.co/9EO3BU4jCx, https://t.co/vqTghjn5bf | AI Agents| 15x Patents | 7x Author | Building for Growth",
    "entities": {
     "description": {
      "urls": [
       {
        "displayUrl": "the-campfire.dev",
        "expandedUrl": "https://the-campfire.dev",
        "indices": [
         29,
         52
        ],
        "url": "https://t.co/ag2bhjo5r8"
       },
       {
        "displayUrl": "ephemeramail.com",
        "expandedUrl": "https://ephemeramail.com",
        "indices": [
         54,
         77
        ],
        "url": "https://t.co/9EO3BU4jCx"
       },
       {
        "displayUrl": "hydranode.ai",
        "expandedUrl": "https://hydranode.ai",
        "indices": [
         79,
         102
        ],
        "url": "https://t.co/vqTghjn5bf"
       }
      ]
     },
     "url": {
      "urls": [
       {
        "displayUrl": "the-campfire.dev",
        "expandedUrl": "https://the-campfire.dev/",
        "indices": [
         0,
         23
        ],
        "url": "https://t.co/bvo0IsCYRp"
       }
      ]
     }
    }
   },
   "hasGraduatedAccess": true,
   "parodyCommentaryFanLabel": "None",
   "professional": {
    "category": [
     {
      "iconName": "IconBriefcaseStroke",
      "id": 958,
      "name": "Entrepreneur"
     }
    ],
    "professionalType": "Creator",
    "restId": "1611459703376060420"
   },
   "profileDescriptionLanguage": "en",
   "profileImageShape": "Circle",
   "profileTranslatorType": "none",
   "superFollowEligible": false
  },
  "quotedTweet": {
   "id": "2093291401496932832",
   "text": "Perplexity changed how we searched the web. But now ChatGPT, Google, and a new generation of AI agents are coming for the same territory. So what happens to Perplexity now? https://t.co/ceEb4v23w2",
   "retweetCount": 2,
   "replyCount": 15,
   "likeCount": 14,
   "quoteCount": 0,
   "viewCount": 2129,
   "bookmarkCount": 0,
   "article": {
    "coverMedia": {
     "id": "QXBpTWVkaWE6DAAFCgABHQzc31QagAAKAAIcWZP92lagAAAA",
     "mediaId": "2093290778567278592",
     "mediaInfo": {
      "__typename": "ApiImage",
      "colorInfo": {
       "palette": [
        {
         "percentage": 91.87,
         "rgb": {
          "blue": 255,
          "green": 255,
          "red": 255
         }
        },
        {
         "percentage": 5.13,
         "rgb": {
          "blue": 78,
          "green": 76,
          "red": 31
         }
        },
        {
         "percentage": 2.13,
         "rgb": {
          "blue": 175,
          "green": 167,
          "red": 84
         }
        },
        {
         "percentage": 0.49,
         "rgb": {
          "blue": 146,
          "green": 143,
          "red": 115
         }
        }
       ]
      },
      "originalImgHeight": 346,
      "originalImgUrl": "https://pbs.twimg.com/media/HQzc31QagAATk5C.jpg",
      "originalImgWidth": 866
     },
     "mediaKey": "3_2093290778567278592"
    },
    "coverMediaUrl": "https://pbs.twimg.com/media/HQzc31QagAATk5C.jpg",
    "id": "2093290613471166464",
    "lifecycleState": {
     "modifiedAtSecs": 1787914535
    },
    "metadata": {
     "firstPublishedAtSecs": 1787914535
    },
    "previewText": "Perplexity was one of those AI products that made people stop and think, “Wait, why am I still searching the old way?” Its pitch was simple. Instead of throwing a list of links at you and making you",
    "title": "Perplexity Was the Future of Search. What Happened?"
   },
   "card": {
    "id": "https://t.co/ceEb4v23w2"
   },
   "conversationId": "2093291401496932832",
   "displayTextRange": [
    0,
    196
   ],
   "edit": {
    "editableUntilMsecs": "1787918136000",
    "editTweetIds": [
     "2093291401496932832"
    ]
   },
   "entities": {
    "hashtags": [],
    "urls": [
     {
      "displayUrl": "x.com/i/article/2093…",
      "expandedUrl": "https://x.com/i/article/2093290613471166464",
      "indices": [
       173,
       196
      ],
      "url": "https://t.co/ceEb4v23w2"
     }
    ],
    "userMentions": []
   },
   "isNoteTweet": false,
   "isQuoteStatus": false,
   "isReply": false,
   "isRetweet": false,
   "isTranslatable": false,
   "lang": "en",
   "possiblySensitive": false,
   "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
   "type": "tweet",
   "url": "https://x.com/_ak_111/status/2093291401496932832",
   "viewState": "EnabledWithCount",
   "createdAt": "Fri Aug 28 10:55:36 +0000 2026",
   "author": {
    "id": "2042826624488349696",
    "username": "_ak_111",
    "name": "Jaysen♨️",
    "followers": 749,
    "following": 269,
    "verified": true,
    "isBlueVerified": true,
    "isVerified": true,
    "profilePicture": "https://pbs.twimg.com/profile_images/2072334961181655041/AN0ljQxX_normal.jpg",
    "description": "18 | AI enthusiast",
    "location": "Kolhapur, india",
    "createdAt": "Sat Apr 11 04:46:52 +0000 2026",
    "statusesCount": 2606,
    "mediaCount": 466,
    "protected": false,
    "url": "https://jaysen67.substack.com",
    "favouritesCount": 3837,
    "isTranslator": false,
    "possiblySensitive": false,
    "pinnedTweetIds": [
     "2095933436196425968"
    ],
    "verifiedType": "Blue",
    "profileBio": {
     "description": "18 | AI enthusiast",
     "entities": {
      "description": {},
      "url": {
       "urls": [
        {
         "displayUrl": "jaysen67.substack.com",
         "expandedUrl": "https://jaysen67.substack.com",
         "indices": [
          0,
          23
         ],
         "url": "https://t.co/pQJ0UZPsup"
        }
       ]
      }
     }
    },
    "hasGraduatedAccess": true,
    "parodyCommentaryFanLabel": "None",
    "professional": {
     "category": [
      {
       "iconName": "IconBriefcaseStroke",
       "id": 144,
       "name": "Education"
      }
     ],
     "professionalType": "Creator",
     "restId": "2047254622801408148"
    },
    "profileImageShape": "Circle",
    "profileTranslatorType": "none",
    "superFollowEligible": false
   }
  },
  "tweetUrl": "https://x.com/stretchcloud/status/2096838373524812149",
  "twitterUrl": "https://twitter.com/stretchcloud/status/2096838373524812149",
  "authorId": "266259493",
  "authorUsername": "stretchcloud",
  "authorName": "Prasenjit Sarkar",
  "authorFollowers": 2989,
  "authorFollowing": 972,
  "authorVerified": true,
  "authorIsBlueVerified": true,
  "authorIsVerified": true,
  "authorProfilePicture": "https://pbs.twimg.com/profile_images/1966979113316409344/BkqBhdQb_normal.jpg",
  "authorCoverPicture": "https://pbs.twimg.com/profile_banners/266259493/1466546448",
  "authorProfileBannerUrl": "https://pbs.twimg.com/profile_banners/266259493/1466546448",
  "authorDescription": "Founder & Builder | Building https://t.co/ag2bhjo5r8, https://t.co/9EO3BU4jCx, https://t.co/vqTghjn5bf | AI Agents| 15x Patents | 7x Author | Building for Growth",
  "authorLocation": "London, England",
  "authorCreatedAt": "Mon Mar 14 23:22:17 +0000 2011",
  "authorStatusesCount": 13292,
  "authorMediaCount": 1097,
  "authorProtected": false,
  "authorUrl": "https://the-campfire.dev/",
  "authorFavouritesCount": 3123,
  "authorIsTranslator": false,
  "authorPossiblySensitive": false,
  "authorVerifiedType": "Blue",
  "authorProfileBio": {
   "description": "Founder & Builder | Building https://t.co/ag2bhjo5r8, https://t.co/9EO3BU4jCx, https://t.co/vqTghjn5bf | AI Agents| 15x Patents | 7x Author | Building for Growth",
   "entities": {
    "description": {
     "urls": [
      {
       "displayUrl": "the-campfire.dev",
       "expandedUrl": "https://the-campfire.dev",
       "indices": [
        29,
        52
       ],
       "url": "https://t.co/ag2bhjo5r8"
      },
      {
       "displayUrl": "ephemeramail.com",
       "expandedUrl": "https://ephemeramail.com",
       "indices": [
        54,
        77
       ],
       "url": "https://t.co/9EO3BU4jCx"
      },
      {
       "displayUrl": "hydranode.ai",
       "expandedUrl": "https://hydranode.ai",
       "indices": [
        79,
        102
       ],
       "url": "https://t.co/vqTghjn5bf"
      }
     ]
    },
    "url": {
     "urls": [
      {
       "displayUrl": "the-campfire.dev",
       "expandedUrl": "https://the-campfire.dev/",
       "indices": [
        0,
        23
       ],
       "url": "https://t.co/bvo0IsCYRp"
      }
     ]
    }
   }
  },
  "authorHasGraduatedAccess": true,
  "authorParodyCommentaryFanLabel": "None",
  "authorProfessional": {
   "category": [
    {
     "iconName": "IconBriefcaseStroke",
     "id": 958,
     "name": "Entrepreneur"
    }
   ],
   "professionalType": "Creator",
   "restId": "1611459703376060420"
  },
  "authorProfileDescriptionLanguage": "en",
  "authorProfileImageShape": "Circle",
  "authorProfileTranslatorType": "none",
  "authorSuperFollowEligible": false,
  "media": []
 }
]